YARN-7289. Application lifetime does not work with FairScheduler. Contributed by Miklos Szegedi.

This commit is contained in:
Rohith Sharma K S 2017-10-27 22:46:38 +05:30
parent 792388e1c0
commit 5c799ecf09
2 changed files with 50 additions and 15 deletions

View File

@ -1797,4 +1797,10 @@ long getNMHeartbeatInterval() {
ReadLock getSchedulerReadLock() { ReadLock getSchedulerReadLock() {
return this.readLock; return this.readLock;
} }
@Override
public long checkAndGetApplicationLifetime(String queueName, long lifetime) {
// Lifetime is the application lifetime by default.
return lifetime;
}
} }

View File

@ -22,7 +22,9 @@
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -54,6 +56,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.util.Times;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
@ -61,19 +64,36 @@
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
/** /**
* Test class for application life time monitor feature test. * Test class for application life time monitor feature test.
*/ */
@RunWith(Parameterized.class)
public class TestApplicationLifetimeMonitor { public class TestApplicationLifetimeMonitor {
private YarnConfiguration conf; private YarnConfiguration conf;
@Parameterized.Parameters
public static Collection<Object[]> data() {
Collection<Object[]> params = new ArrayList<Object[]>();
params.add(new Object[]{CapacityScheduler.class});
params.add(new Object[]{FairScheduler.class});
return params;
}
private Class scheduler;
public TestApplicationLifetimeMonitor(Class schedulerParameter) {
scheduler = schedulerParameter;
}
@Before @Before
public void setup() throws IOException { public void setup() throws IOException {
conf = new YarnConfiguration(); conf = new YarnConfiguration();
// Always run for CS, since other scheduler do not support this. // Always run for CS, since other scheduler do not support this.
conf.setClass(YarnConfiguration.RM_SCHEDULER, conf.setClass(YarnConfiguration.RM_SCHEDULER,
CapacityScheduler.class, ResourceScheduler.class); scheduler, ResourceScheduler.class);
Logger rootLogger = LogManager.getRootLogger(); Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG); rootLogger.setLevel(Level.DEBUG);
UserGroupInformation.setConfiguration(conf); UserGroupInformation.setConfiguration(conf);
@ -82,15 +102,20 @@ public void setup() throws IOException {
} }
@Test(timeout = 60000) @Test(timeout = 60000)
public void testApplicationLifetimeMonitor() throws Exception { public void testApplicationLifetimeMonitor()
throws Exception {
MockRM rm = null; MockRM rm = null;
try { try {
long maxLifetime = 30L; long maxLifetime = 30L;
long defaultLifetime = 15L; long defaultLifetime = 15L;
YarnConfiguration newConf = YarnConfiguration newConf;
if (scheduler.equals(CapacityScheduler.class)) {
// Since there is limited lifetime monitoring support in fair scheduler
// it does not need queue setup
conf =
new YarnConfiguration(setUpCSQueue(maxLifetime, defaultLifetime)); new YarnConfiguration(setUpCSQueue(maxLifetime, defaultLifetime));
conf = new YarnConfiguration(newConf); }
rm = new MockRM(conf); rm = new MockRM(conf);
rm.start(); rm.start();
@ -172,17 +197,21 @@ public void testApplicationLifetimeMonitor() throws Exception {
Assert.assertTrue("Application killed before lifetime value", Assert.assertTrue("Application killed before lifetime value",
app2.getFinishTime() > afterUpdate); app2.getFinishTime() > afterUpdate);
if (scheduler.equals(CapacityScheduler.class)) {
// Supported only on capacity scheduler
rm.waitForState(app3.getApplicationId(), RMAppState.KILLED); rm.waitForState(app3.getApplicationId(), RMAppState.KILLED);
// app4 submitted exceeding queue max lifetime, so killed after queue max // app4 submitted exceeding queue max lifetime,
// lifetime. // so killed after queue max lifetime.
rm.waitForState(app4.getApplicationId(), RMAppState.KILLED); rm.waitForState(app4.getApplicationId(), RMAppState.KILLED);
long totalTimeRun = (app4.getFinishTime() - app4.getSubmitTime()) / 1000; long totalTimeRun =
(app4.getFinishTime() - app4.getSubmitTime()) / 1000;
Assert.assertTrue("Application killed before lifetime value", Assert.assertTrue("Application killed before lifetime value",
totalTimeRun > maxLifetime); totalTimeRun > maxLifetime);
Assert.assertTrue( Assert.assertTrue(
"Application killed before lifetime value " + totalTimeRun, "Application killed before lifetime value " + totalTimeRun,
totalTimeRun < maxLifetime + 10L); totalTimeRun < maxLifetime + 10L);
}
} finally { } finally {
stopRM(rm); stopRM(rm);
} }