big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18455. S3A prefetching executor should be closed (#4879)

Browse Source 
follow-on patch to HADOOP-18186. 

Contributed by: Viraj Jasani
This commit is contained in:
Viraj Jasani 2022-09-21 11:52:41 -07:00 committed by GitHub
parent 740e1ef357
commit 084b68e380
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 29 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ExecutorServiceFuturePool.java
View File

@ -22,8 +22,13 @@
import java.util.Locale;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import org.slf4j.Logger;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
/**
* A FuturePool implementation backed by a java.util.concurrent.ExecutorService.
*
@ -37,7 +42,8 @@
*
*/
public class ExecutorServiceFuturePool {
private ExecutorService executor;
private final ExecutorService executor;
public ExecutorServiceFuturePool(ExecutorService executor) {
this.executor = executor;
@ -64,6 +70,18 @@ public Future<Void> executeRunnable(final Runnable r) {
return (Future<Void>) executor.submit(r::run);
}
/**
* Utility to shutdown the {@link ExecutorService} used by this class. Will wait up to a
* certain timeout for the ExecutorService to gracefully shutdown.
*
* @param logger Logger
* @param timeout the maximum time to wait
* @param unit the time unit of the timeout argument
*/
public void shutdown(Logger logger, long timeout, TimeUnit unit) {
HadoopExecutors.shutdown(executor, logger, timeout, unit);
}
public String toString() {
return String.format(Locale.ROOT, "ExecutorServiceFuturePool(executor=%s)", executor);
}

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java
View File

@ -115,9 +115,8 @@ public static void shutdown(ExecutorService executorService, Logger logger,
try {
executorService.shutdown();
logger.debug(
"Gracefully shutting down executor service. Waiting max {} {}",
timeout, unit);
logger.debug("Gracefully shutting down executor service {}. Waiting max {} {}",
executorService, timeout, unit);
if (!executorService.awaitTermination(timeout, unit)) {
logger.debug(
"Executor service has not shutdown yet. Forcing. "

4
hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
View File

@ -59,6 +59,10 @@
<Method name="openFileWithOptions"/>
<Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"/>
</Match>
<Match>
<Class name="org.apache.hadoop.fs.s3a.S3AFileSystem"/>
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
<Match>
<Class name="org.apache.hadoop.fs.s3a.s3guard.S3GuardTool$BucketInfo"/>
<Method name="run"/>

10
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -633,17 +633,11 @@ public void initialize(URI name, Configuration originalConf)
// amazon client exception: stop all services then throw the translation
cleanupWithLogger(LOG, span);
stopAllServices();
if (this.futurePool != null) {
this.futurePool = null;
}
throw translateException("initializing ", new Path(name), e);
} catch (IOException | RuntimeException e) {
// other exceptions: stop the services.
cleanupWithLogger(LOG, span);
stopAllServices();
if (this.futurePool != null) {
this.futurePool = null;
}
throw e;
}
}
@ -4038,6 +4032,10 @@ protected synchronized void stopAllServices() {
HadoopExecutors.shutdown(unboundedThreadPool, LOG,
THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
unboundedThreadPool = null;
if (futurePool != null) {
futurePool.shutdown(LOG, THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
futurePool = null;
}
// other services are shutdown.
cleanupWithLogger(LOG,
instrumentation,