YARN-3305. Normalize AM resource request on app submission. Contributed by Rohith Sharmaks

This commit is contained in:
Jian He 2015-03-17 13:49:59 -07:00
parent 32b4330456
commit 968425e9f7
5 changed files with 47 additions and 3 deletions

View File

@ -66,8 +66,11 @@ Release 2.8.0 - UNRELEASED
BUG FIXES BUG FIXES
YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena
via devaraj) via devaraj)
YARN-3305. Normalize AM resource request on app submission. (Rohith Sharmaks
via jianhe)
Release 2.7.0 - UNRELEASED Release 2.7.0 - UNRELEASED

View File

@ -390,7 +390,11 @@ private ResourceRequest validateAndCreateResourceRequest(
+ " for application " + submissionContext.getApplicationId(), e); + " for application " + submissionContext.getApplicationId(), e);
throw e; throw e;
} }
SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(),
scheduler.getClusterResource(),
scheduler.getMinimumResourceCapability(),
scheduler.getMaximumResourceCapability(),
scheduler.getMinimumResourceCapability());
return amReq; return amReq;
} }

View File

@ -67,6 +67,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
@ -604,6 +605,10 @@ private static ResourceScheduler mockResourceScheduler() {
when(scheduler.getMaximumResourceCapability()).thenReturn( when(scheduler.getMaximumResourceCapability()).thenReturn(
Resources.createResource( Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
ResourceCalculator rs = mock(ResourceCalculator.class);
when(scheduler.getResourceCalculator()).thenReturn(rs);
return scheduler; return scheduler;
} }

View File

@ -146,6 +146,7 @@
import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.UTCClock; import org.apache.hadoop.yarn.util.UTCClock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
@ -1289,6 +1290,10 @@ private static YarnScheduler mockYarnScheduler() {
Arrays.asList(getApplicationAttemptId(103))); Arrays.asList(getApplicationAttemptId(103)));
ApplicationAttemptId attemptId = getApplicationAttemptId(1); ApplicationAttemptId attemptId = getApplicationAttemptId(1);
when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null); when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
ResourceCalculator rs = mock(ResourceCalculator.class);
when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
return yarnScheduler; return yarnScheduler;
} }

View File

@ -2543,6 +2543,33 @@ public void testParentQueueMaxCapsAreRespected() throws Exception {
+ "queue-a's max capacity will be violated if container allocated"); + "queue-a's max capacity will be violated if container allocated");
} }
// Test verifies AM Used resource for LeafQueue when AM ResourceRequest is
// lesser than minimumAllocation
@Test(timeout = 30000)
public void testAMUsedResource() throws Exception {
MockRM rm = setUpMove();
Configuration conf = rm.getConfig();
int minAllocMb =
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
int amMemory = 50;
assertTrue("AM memory is greater than or equql to minAllocation",
amMemory < minAllocMb);
Resource minAllocResource = Resource.newInstance(minAllocMb, 1);
String queueName = "a1";
RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName);
assertEquals("RMApp does not containes minimum allocation",
minAllocResource, rmApp.getAMResourceRequest().getCapability());
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
LeafQueue queueA =
(LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName);
assertEquals("Minimum Resource for AM is incorrect", minAllocResource,
queueA.getUser("user_0").getResourceUsage().getAMUsed());
rm.stop();
}
private void setMaxAllocMb(Configuration conf, int maxAllocMb) { private void setMaxAllocMb(Configuration conf, int maxAllocMb) {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
maxAllocMb); maxAllocMb);