diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 0f51fef58f..173ddcbeec 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -158,6 +158,9 @@ public abstract class TaskAttemptImpl implements org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt, EventHandler { + @VisibleForTesting + protected final static Map RESOURCE_REQUEST_CACHE + = new HashMap<>(); static final Counters EMPTY_COUNTERS = new Counters(); private static final Logger LOG = LoggerFactory.getLogger(TaskAttemptImpl.class); @@ -172,7 +175,7 @@ public abstract class TaskAttemptImpl implements private final Clock clock; private final org.apache.hadoop.mapred.JobID oldJobId; private final TaskAttemptListener taskAttemptListener; - private final Resource resourceCapability; + private Resource resourceCapability; protected Set dataLocalHosts; protected Set dataLocalRacks; private final List diagnostics = new ArrayList(); @@ -707,6 +710,10 @@ private void populateResourceCapability(TaskType taskType) { getResourceTypePrefix(taskType); boolean memorySet = false; boolean cpuVcoresSet = false; + if (RESOURCE_REQUEST_CACHE.get(taskType) != null) { + resourceCapability = RESOURCE_REQUEST_CACHE.get(taskType); + return; + } if (resourceTypePrefix != null) { List resourceRequests = ResourceUtils.getRequestedResourcesFromConfig(conf, @@ -767,6 +774,9 @@ private void populateResourceCapability(TaskType taskType) { if (!cpuVcoresSet) { this.resourceCapability.setVirtualCores(getCpuRequired(conf, taskType)); } + RESOURCE_REQUEST_CACHE.put(taskType, resourceCapability); + LOG.info("Resource capability of task type {} is set to {}", + taskType, resourceCapability); } private String getCpuVcoresKey(TaskType taskType) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java index a5f0f7b788..96d1ebe9a0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider; import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -155,6 +156,11 @@ public static void setupBeforeClass() { ResourceUtils.resetResourceTypes(new Configuration()); } + @Before + public void before() { + TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear(); + } + @After public void tearDown() { ResourceUtils.resetResourceTypes(new Configuration()); @@ -1721,6 +1727,7 @@ public void testReducerMemoryRequestOverriding() { TestAppender testAppender = new TestAppender(); final Logger logger = Logger.getLogger(TaskAttemptImpl.class); try { + TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear(); logger.addAppender(testAppender); EventHandler eventHandler = mock(EventHandler.class); Clock clock = SystemClock.getInstance();