diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e039a8ff24..12b9ce0bc0 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -702,6 +702,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3775. Change MiniYarnCluster to escape special chars in testname. (Hitesh Shah via mahadev) + MAPREDUCE-3765. FifoScheduler does not respect yarn.scheduler.fifo.minimum- + allocation-mb setting (Hitesh Shah via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 4b7d45f66a..4b17c1e943 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -161,6 +161,12 @@ + + + + + + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index ecdb758923..3d904f3a8c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -230,7 +230,7 @@ public Allocation allocate( } // Sanity check - SchedulerUtils.normalizeRequests(ask, MINIMUM_MEMORY); + SchedulerUtils.normalizeRequests(ask, minimumAllocation.getMemory()); // Release containers for (ContainerId releasedContainer : release) { @@ -592,7 +592,7 @@ private synchronized void nodeUpdate(RMNode rmNode, minimumAllocation)) { LOG.debug("Node heartbeat " + rmNode.getNodeID() + " available resource = " + node.getAvailableResource()); - + assignContainers(node); LOG.debug("Node after allocation " + rmNode.getNodeID() + " resource = " diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 349a2cacbe..2cca6f09ad 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -24,38 +24,22 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.AMResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; -import org.junit.After; -import org.junit.Before; import org.junit.Test; public class TestFifoScheduler { private static final Log LOG = LogFactory.getLog(TestFifoScheduler.class); - private ResourceManager resourceManager = null; - - @Before - public void setUp() throws Exception { - Store store = StoreFactory.getStore(new Configuration()); - resourceManager = new ResourceManager(store); - resourceManager.init(new Configuration()); - } - - @After - public void tearDown() throws Exception { - } + private final int GB = 1024; @Test public void test() throws Exception { @@ -63,7 +47,6 @@ public void test() throws Exception { rootLogger.setLevel(Level.DEBUG); MockRM rm = new MockRM(); rm.start(); - int GB = 1024; MockNM nm1 = rm.registerNode("h1:1234", 6 * GB); MockNM nm2 = rm.registerNode("h2:5678", 4 * GB); @@ -146,8 +129,48 @@ public void test() throws Exception { rm.stop(); } + private void testMinimumAllocation(YarnConfiguration conf) + throws Exception { + MockRM rm = new MockRM(conf); + rm.start(); + + // Register node1 + MockNM nm1 = rm.registerNode("h1:1234", 6 * GB); + + // Submit an application + RMApp app1 = rm.submitApp(256); + + // kick the scheduling + nm1.nodeHeartbeat(true); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); + am1.registerAppAttempt(); + SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( + nm1.getNodeId()); + + int checkAlloc = + conf.getInt("yarn.scheduler.fifo.minimum-allocation-mb", GB); + Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemory()); + + rm.stop(); + } + + @Test + public void testDefaultMinimumAllocation() throws Exception { + testMinimumAllocation(new YarnConfiguration()); + } + + @Test + public void testNonDefaultMinimumAllocation() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt("yarn.scheduler.fifo.minimum-allocation-mb", 512); + testMinimumAllocation(conf); + } + public static void main(String[] args) throws Exception { TestFifoScheduler t = new TestFifoScheduler(); t.test(); + t.testDefaultMinimumAllocation(); + t.testNonDefaultMinimumAllocation(); } }