diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1982688b86..3507420f12 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -318,6 +318,9 @@ Release 2.7.0 - UNRELEASED YARN-3236. Cleanup RMAuthenticationFilter#AUTH_HANDLER_PROPERTY. (zhihai xu via xgong) + YARN-2797. TestWorkPreservingRMRestart should use ParametrizedSchedulerTestBase + (Karthik Kambatla via xgong) + OPTIMIZATIONS YARN-2990. FairScheduler's delay-scheduling always waits for node-local and diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java index cfd16001a3..b0998368bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java @@ -83,10 +83,22 @@ private void configureFairScheduler(YarnConfiguration conf) throws IOException { out.println(""); out.println(""); out.println("-1.0"); + out.println("fair"); + out.println(""); + out.println(" drf"); + out.println(" 1.0"); + out.println(" 100"); + out.println(" 120"); + out.println(" .5"); + out.println(""); out.println(""); out.close(); conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); } + + public SchedulerType getSchedulerType() { + return schedulerType; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index a9caf77d36..30334965bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -97,23 +97,24 @@ @SuppressWarnings({"rawtypes", "unchecked"}) @RunWith(value = Parameterized.class) -public class TestWorkPreservingRMRestart { +public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase { private YarnConfiguration conf; - private Class schedulerClass; MockRM rm1 = null; MockRM rm2 = null; + public TestWorkPreservingRMRestart(SchedulerType type) { + super(type); + } + @Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); - conf = new YarnConfiguration(); + conf = getConf(); UserGroupInformation.setConfiguration(conf); conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); - conf.setClass(YarnConfiguration.RM_SCHEDULER, schedulerClass, - ResourceScheduler.class); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true); conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0); DefaultMetricsSystem.setMiniClusterMode(true); @@ -129,16 +130,6 @@ public void tearDown() { } } - @Parameterized.Parameters - public static Collection getTestParameters() { - return Arrays.asList(new Object[][] { { CapacityScheduler.class }, - { FifoScheduler.class }, {FairScheduler.class } }); - } - - public TestWorkPreservingRMRestart(Class schedulerClass) { - this.schedulerClass = schedulerClass; - } - // Test common scheduler state including SchedulerAttempt, SchedulerNode, // AppSchedulingInfo can be reconstructed via the container recovery reports // on NM re-registration. @@ -159,9 +150,6 @@ public void testSchedulerRecovery() throws Exception { MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); rm1 = new MockRM(conf, memStore); - if (schedulerClass.equals(FairScheduler.class)) { - initFairScheduler(rm1); - } rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); @@ -174,9 +162,6 @@ public void testSchedulerRecovery() throws Exception { // Re-start RM rm2 = new MockRM(conf, memStore); - if (schedulerClass.equals(FairScheduler.class)) { - initFairScheduler(rm2); - } rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); // recover app @@ -249,11 +234,9 @@ public void testSchedulerRecovery() throws Exception { SchedulerApplication schedulerApp = schedulerApps.get(recoveredApp1.getApplicationId()); - if (schedulerClass.equals(CapacityScheduler.class)) { + if (getSchedulerType() == SchedulerType.CAPACITY) { checkCSQueue(rm2, schedulerApp, nmResource, nmResource, usedResources, 2); - } else if (schedulerClass.equals(FifoScheduler.class)) { - checkFifoQueue(rm2, schedulerApp, usedResources, availableResources); - } else if (schedulerClass.equals(FairScheduler.class)) { + } else { checkFSQueue(rm2, schedulerApp, usedResources, availableResources); } @@ -324,25 +307,6 @@ private void checkCSLeafQueue(MockRM rm, .getUsed()); } - private void checkFifoQueue(ResourceManager rm, - SchedulerApplication schedulerApp, Resource usedResources, - Resource availableResources) throws Exception { - FifoScheduler scheduler = (FifoScheduler) rm.getResourceScheduler(); - // ************ check cluster used Resources ******** - assertEquals(usedResources, scheduler.getUsedResource()); - - // ************ check app headroom **************** - SchedulerApplicationAttempt schedulerAttempt = - schedulerApp.getCurrentAppAttempt(); - assertEquals(availableResources, schedulerAttempt.getHeadroom()); - - // ************ check queue metrics **************** - QueueMetrics queueMetrics = scheduler.getRootQueueMetrics(); - assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(), - availableResources.getVirtualCores(), usedResources.getMemory(), - usedResources.getVirtualCores()); - } - private void checkFSQueue(ResourceManager rm, SchedulerApplication schedulerApp, Resource usedResources, Resource availableResources) throws Exception { @@ -379,29 +343,6 @@ private void checkFSQueue(ResourceManager rm, usedResources.getVirtualCores()); } - private void initFairScheduler(ResourceManager rm) throws IOException { - FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); - String testDir = - new File( - System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); - String allocFile = new File(testDir, "test-queues").getAbsolutePath(); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); - - PrintWriter out = new PrintWriter(new FileWriter(allocFile)); - out.println(""); - out.println(""); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" 1.0"); - out.println(" 100"); - out.println(" 120"); - out.println(" .5"); - out.println(""); - out.println(""); - out.close(); - } - // create 3 container reports for AM public static List createNMContainerStatusForApp(MockAM am) { @@ -468,7 +409,7 @@ private void setupQueueConfigurationOnlyA( // 10. Assert each user's consumption inside the queue. @Test (timeout = 30000) public void testCapacitySchedulerRecovery() throws Exception { - if (!schedulerClass.equals(CapacityScheduler.class)) { + if (getSchedulerType() != SchedulerType.CAPACITY) { return; } conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true); @@ -587,7 +528,7 @@ public void testCapacitySchedulerRecovery() throws Exception { //3. Verify that the expected exception was thrown @Test (timeout = 30000, expected = QueueNotFoundException.class) public void testCapacitySchedulerQueueRemovedRecovery() throws Exception { - if (!schedulerClass.equals(CapacityScheduler.class)) { + if (getSchedulerType() != SchedulerType.CAPACITY) { throw new QueueNotFoundException("Dummy"); } conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);