YARN-7289. Application lifetime does not work with FairScheduler. Contributed by Miklos Szegedi.
This commit is contained in:
parent
792388e1c0
commit
5c799ecf09
@ -1797,4 +1797,10 @@ long getNMHeartbeatInterval() {
|
|||||||
ReadLock getSchedulerReadLock() {
|
ReadLock getSchedulerReadLock() {
|
||||||
return this.readLock;
|
return this.readLock;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long checkAndGetApplicationLifetime(String queueName, long lifetime) {
|
||||||
|
// Lifetime is the application lifetime by default.
|
||||||
|
return lifetime;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -22,7 +22,9 @@
|
|||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@ -54,6 +56,7 @@
|
|||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||||
import org.apache.hadoop.yarn.util.Times;
|
import org.apache.hadoop.yarn.util.Times;
|
||||||
import org.apache.log4j.Level;
|
import org.apache.log4j.Level;
|
||||||
import org.apache.log4j.LogManager;
|
import org.apache.log4j.LogManager;
|
||||||
@ -61,19 +64,36 @@
|
|||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.junit.runner.RunWith;
|
||||||
|
import org.junit.runners.Parameterized;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test class for application life time monitor feature test.
|
* Test class for application life time monitor feature test.
|
||||||
*/
|
*/
|
||||||
|
@RunWith(Parameterized.class)
|
||||||
public class TestApplicationLifetimeMonitor {
|
public class TestApplicationLifetimeMonitor {
|
||||||
private YarnConfiguration conf;
|
private YarnConfiguration conf;
|
||||||
|
|
||||||
|
@Parameterized.Parameters
|
||||||
|
public static Collection<Object[]> data() {
|
||||||
|
Collection<Object[]> params = new ArrayList<Object[]>();
|
||||||
|
params.add(new Object[]{CapacityScheduler.class});
|
||||||
|
params.add(new Object[]{FairScheduler.class});
|
||||||
|
return params;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Class scheduler;
|
||||||
|
|
||||||
|
public TestApplicationLifetimeMonitor(Class schedulerParameter) {
|
||||||
|
scheduler = schedulerParameter;
|
||||||
|
}
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setup() throws IOException {
|
public void setup() throws IOException {
|
||||||
conf = new YarnConfiguration();
|
conf = new YarnConfiguration();
|
||||||
// Always run for CS, since other scheduler do not support this.
|
// Always run for CS, since other scheduler do not support this.
|
||||||
conf.setClass(YarnConfiguration.RM_SCHEDULER,
|
conf.setClass(YarnConfiguration.RM_SCHEDULER,
|
||||||
CapacityScheduler.class, ResourceScheduler.class);
|
scheduler, ResourceScheduler.class);
|
||||||
Logger rootLogger = LogManager.getRootLogger();
|
Logger rootLogger = LogManager.getRootLogger();
|
||||||
rootLogger.setLevel(Level.DEBUG);
|
rootLogger.setLevel(Level.DEBUG);
|
||||||
UserGroupInformation.setConfiguration(conf);
|
UserGroupInformation.setConfiguration(conf);
|
||||||
@ -82,15 +102,20 @@ public void setup() throws IOException {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test(timeout = 60000)
|
||||||
public void testApplicationLifetimeMonitor() throws Exception {
|
public void testApplicationLifetimeMonitor()
|
||||||
|
throws Exception {
|
||||||
MockRM rm = null;
|
MockRM rm = null;
|
||||||
try {
|
try {
|
||||||
long maxLifetime = 30L;
|
long maxLifetime = 30L;
|
||||||
long defaultLifetime = 15L;
|
long defaultLifetime = 15L;
|
||||||
|
|
||||||
YarnConfiguration newConf =
|
YarnConfiguration newConf;
|
||||||
|
if (scheduler.equals(CapacityScheduler.class)) {
|
||||||
|
// Since there is limited lifetime monitoring support in fair scheduler
|
||||||
|
// it does not need queue setup
|
||||||
|
conf =
|
||||||
new YarnConfiguration(setUpCSQueue(maxLifetime, defaultLifetime));
|
new YarnConfiguration(setUpCSQueue(maxLifetime, defaultLifetime));
|
||||||
conf = new YarnConfiguration(newConf);
|
}
|
||||||
rm = new MockRM(conf);
|
rm = new MockRM(conf);
|
||||||
rm.start();
|
rm.start();
|
||||||
|
|
||||||
@ -172,17 +197,21 @@ public void testApplicationLifetimeMonitor() throws Exception {
|
|||||||
Assert.assertTrue("Application killed before lifetime value",
|
Assert.assertTrue("Application killed before lifetime value",
|
||||||
app2.getFinishTime() > afterUpdate);
|
app2.getFinishTime() > afterUpdate);
|
||||||
|
|
||||||
|
if (scheduler.equals(CapacityScheduler.class)) {
|
||||||
|
// Supported only on capacity scheduler
|
||||||
rm.waitForState(app3.getApplicationId(), RMAppState.KILLED);
|
rm.waitForState(app3.getApplicationId(), RMAppState.KILLED);
|
||||||
|
|
||||||
// app4 submitted exceeding queue max lifetime, so killed after queue max
|
// app4 submitted exceeding queue max lifetime,
|
||||||
// lifetime.
|
// so killed after queue max lifetime.
|
||||||
rm.waitForState(app4.getApplicationId(), RMAppState.KILLED);
|
rm.waitForState(app4.getApplicationId(), RMAppState.KILLED);
|
||||||
long totalTimeRun = (app4.getFinishTime() - app4.getSubmitTime()) / 1000;
|
long totalTimeRun =
|
||||||
|
(app4.getFinishTime() - app4.getSubmitTime()) / 1000;
|
||||||
Assert.assertTrue("Application killed before lifetime value",
|
Assert.assertTrue("Application killed before lifetime value",
|
||||||
totalTimeRun > maxLifetime);
|
totalTimeRun > maxLifetime);
|
||||||
Assert.assertTrue(
|
Assert.assertTrue(
|
||||||
"Application killed before lifetime value " + totalTimeRun,
|
"Application killed before lifetime value " + totalTimeRun,
|
||||||
totalTimeRun < maxLifetime + 10L);
|
totalTimeRun < maxLifetime + 10L);
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
stopRM(rm);
|
stopRM(rm);
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user