From 92f1f2c2a671ac9b2af8bd066d5434ab541ec8d0 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Sun, 30 Sep 2012 03:32:07 +0000 Subject: [PATCH] YARN-138. Ensure default values for minimum/maximum container sizes is sane. Contributed by Harsh J and Siddharth Seth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1391944 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 6 ++-- .../src/main/resources/yarn-default.xml | 4 +-- .../resourcemanager/TestFifoScheduler.java | 2 +- .../TestAMRMRPCNodeUpdates.java | 8 +++--- .../scheduler/fair/TestFairScheduler.java | 28 +++++++++++-------- 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f09c182cac..448884d755 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -97,6 +97,9 @@ Release 2.0.2-alpha - 2012-09-07 MAPREDUCE-2374. "Text File Busy" errors launching MR tasks. (Andy Isaacson via atm) + YARN-138. Ensure default values for minimum/maximum container sizes is + sane. (harsh & sseth via acmurthy) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 491605e7cf..cbe63fdbd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -114,18 +114,18 @@ public class YarnConfiguration extends Configuration { /** Miniumum memory request grant-able by the RM scheduler. */ public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB = YARN_PREFIX + "scheduler.minimum-allocation-mb"; - public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 128; + public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; /** Maximum memory request grant-able by the RM scheduler. */ public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = YARN_PREFIX + "scheduler.maximum-allocation-mb"; - public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 10240; + public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192; /** Number of threads to handle scheduler interface.*/ public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = RM_PREFIX + "scheduler.client.thread-count"; public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; - + /** The address of the RM web application.*/ public static final String RM_WEBAPP_ADDRESS = RM_PREFIX + "webapp.address"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 6f829be73a..adcf8d54ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -193,7 +193,7 @@ in MBs. Memory requests lower than this won't take effect, and the specified value will get allocated at minimum. yarn.scheduler.minimum-allocation-mb - 128 + 1024 @@ -201,7 +201,7 @@ in MBs. Memory requests higher than this won't take effect, and will get capped to this value. yarn.scheduler.maximum-allocation-mb - 10240 + 8192 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 1fd5d9bb36..968b7096cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -194,7 +194,7 @@ public void testDefaultMinimumAllocation() throws Exception { @Test public void testNonDefaultMinimumAllocation() throws Exception { // Set custom min-alloc to test tweaking it - int allocMB = 512; + int allocMB = 1536; YarnConfiguration conf = new YarnConfiguration(TestFifoScheduler.conf); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, allocMB); // Test for something lesser than this. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java index d709beebeb..d607c0181d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java @@ -90,10 +90,10 @@ private void syncNodeLost(MockNM nm) throws Exception { @Test public void testAMRMUnusableNodes() throws Exception { - MockNM nm1 = rm.registerNode("h1:1234", 5000); - MockNM nm2 = rm.registerNode("h2:1234", 5000); - MockNM nm3 = rm.registerNode("h3:1234", 5000); - MockNM nm4 = rm.registerNode("h4:1234", 5000); + MockNM nm1 = rm.registerNode("h1:1234", 10000); + MockNM nm2 = rm.registerNode("h2:1234", 10000); + MockNM nm3 = rm.registerNode("h3:1234", 10000); + MockNM nm4 = rm.registerNode("h4:1234", 10000); RMApp app1 = rm.submitApp(2000); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 38f7547377..4a26920166 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -225,7 +225,9 @@ public void testSimpleContainerAllocation() { new ArrayList(), new ArrayList()); scheduler.handle(updateEvent); - assertEquals(512, scheduler.getQueueManager().getQueue("queue1"). + // Asked for less than min_allocation. + assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + scheduler.getQueueManager().getQueue("queue1"). getQueueSchedulable().getResourceUsage().getMemory()); NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2, @@ -362,31 +364,35 @@ public void testQueueDemandCalculation() throws Exception { ApplicationAttemptId id22 = createAppAttemptId(2, 2); scheduler.addApplication(id22, "queue2", "user1"); - // First ask, queue1 requests 1024 + int minReqSize = YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB; + + // First ask, queue1 requests 1 large (minReqSize * 2). List ask1 = new ArrayList(); - ResourceRequest request1 = createResourceRequest(1024, "*", 1, 1); + ResourceRequest request1 = createResourceRequest(minReqSize * 2, "*", 1, 1); ask1.add(request1); scheduler.allocate(id11, ask1, new ArrayList()); - // Second ask, queue2 requests 1024 + (2 * 512) + // Second ask, queue2 requests 1 large + (2 * minReqSize) List ask2 = new ArrayList(); - ResourceRequest request2 = createResourceRequest(1024, "foo", 1, 1); - ResourceRequest request3 = createResourceRequest(512, "bar", 1, 2); + ResourceRequest request2 = createResourceRequest(2 * minReqSize, "foo", 1, 1); + ResourceRequest request3 = createResourceRequest(minReqSize, "bar", 1, 2); ask2.add(request2); ask2.add(request3); scheduler.allocate(id21, ask2, new ArrayList()); - // Third ask, queue2 requests 1024 + // Third ask, queue2 requests 1 large List ask3 = new ArrayList(); - ResourceRequest request4 = createResourceRequest(1024, "*", 1, 1); + ResourceRequest request4 = createResourceRequest(2 * minReqSize, "*", 1, 1); ask3.add(request4); scheduler.allocate(id22, ask3, new ArrayList()); scheduler.update(); - assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").getQueueSchedulable().getDemand().getMemory()); - assertEquals(1024 + 1024 + (2 * 512), scheduler.getQueueManager().getQueue("queue2").getQueueSchedulable().getDemand().getMemory()); - + assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("queue1") + .getQueueSchedulable().getDemand().getMemory()); + assertEquals(2 * minReqSize + 2 * minReqSize + (2 * minReqSize), scheduler + .getQueueManager().getQueue("queue2").getQueueSchedulable().getDemand() + .getMemory()); } @Test