big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3090. DeletionService can silently ignore deletion task failures. Contributed by Varun Saxena

Browse Source
This commit is contained in:
Jason Lowe 2015-02-10 16:54:21 +00:00
parent e0ec0718d0
commit 4eb5f7fa32
2 changed files with 38 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -527,6 +527,9 @@ Release 2.7.0 - UNRELEASED
YARN-2971. RM uses conf instead of token service address to renew timeline YARN-2971. RM uses conf instead of token service address to renew timeline
delegation tokens (jeagles) delegation tokens (jeagles)
YARN-3090. DeletionService can silently ignore deletion task failures
(Varun Saxena via jlowe)
Release 2.6.0 - 2014-11-18 Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
View File

@ -29,6 +29,8 @@
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -113,13 +115,13 @@ protected void serviceInit(Configuration conf) throws Exception {
.setNameFormat("DeletionService #%d") .setNameFormat("DeletionService #%d")
.build(); .build();
if (conf != null) { if (conf != null) {
sched = new ScheduledThreadPoolExecutor( sched = new DelServiceSchedThreadPoolExecutor(
conf.getInt(YarnConfiguration.NM_DELETE_THREAD_COUNT, YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT), conf.getInt(YarnConfiguration.NM_DELETE_THREAD_COUNT,
tf); YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT), tf);
debugDelay = conf.getInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0); debugDelay = conf.getInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0);
} else { } else {
sched = new ScheduledThreadPoolExecutor(YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT, sched = new DelServiceSchedThreadPoolExecutor(
tf); YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT, tf);
} }
sched.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); sched.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
sched.setKeepAliveTime(60L, SECONDS); sched.setKeepAliveTime(60L, SECONDS);
@ -155,6 +157,34 @@ public boolean isTerminated() {
return getServiceState() == STATE.STOPPED && sched.isTerminated(); return getServiceState() == STATE.STOPPED && sched.isTerminated();
} }
private static class DelServiceSchedThreadPoolExecutor extends
ScheduledThreadPoolExecutor {
public DelServiceSchedThreadPoolExecutor(int corePoolSize,
ThreadFactory threadFactory) {
super(corePoolSize, threadFactory);
}
@Override
protected void afterExecute(Runnable task, Throwable exception) {
if (task instanceof FutureTask<?>) {
FutureTask<?> futureTask = (FutureTask<?>) task;
if (!futureTask.isCancelled()) {
try {
futureTask.get();
} catch (ExecutionException ee) {
exception = ee.getCause();
} catch (InterruptedException ie) {
exception = ie;
}
}
}
if (exception != null) {
LOG.error("Exception during execution of task in DeletionService",
exception);
}
}
}
public static class FileDeletionTask implements Runnable { public static class FileDeletionTask implements Runnable {
public static final int INVALID_TASK_ID = -1; public static final int INVALID_TASK_ID = -1;
private int taskId; private int taskId;