YARN-3686. CapacityScheduler should trim default_node_label_expression. (Sunil G via wangda)

This commit is contained in:
Wangda Tan 2015-05-26 15:58:47 -07:00
parent 7dba7005b7
commit cdbd66be11
6 changed files with 79 additions and 3 deletions

View File

@ -567,6 +567,9 @@ Release 2.7.1 - UNRELEASED
YARN-2238. filtering on UI sticks even if I move away from the page. YARN-2238. filtering on UI sticks even if I move away from the page.
(Jian He via xgong) (Jian He via xgong)
YARN-3686. CapacityScheduler should trim default_node_label_expression.
(Sunil G via wangda)
Release 2.7.0 - 2015-04-20 Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -358,7 +358,7 @@ public Set<String> getAccessibleNodeLabels() {
public String getDefaultNodeLabelExpression() { public String getDefaultNodeLabelExpression() {
QueueInfoProtoOrBuilder p = viaProto ? proto : builder; QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
return (p.hasDefaultNodeLabelExpression()) ? p return (p.hasDefaultNodeLabelExpression()) ? p
.getDefaultNodeLabelExpression() : null; .getDefaultNodeLabelExpression().trim() : null;
} }
@Override @Override

View File

@ -193,7 +193,7 @@ public String getNodeLabelExpression() {
if (!p.hasNodeLabelExpression()) { if (!p.hasNodeLabelExpression()) {
return null; return null;
} }
return (p.getNodeLabelExpression()); return (p.getNodeLabelExpression().trim());
} }
@Override @Override

View File

@ -524,7 +524,12 @@ public float getLabeledQueueMaximumCapacity(String queue, String label) {
} }
public String getDefaultNodeLabelExpression(String queue) { public String getDefaultNodeLabelExpression(String queue) {
return get(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION); String defaultLabelExpression = get(getQueuePrefix(queue)
+ DEFAULT_NODE_LABEL_EXPRESSION);
if (defaultLabelExpression == null) {
return null;
}
return defaultLabelExpression.trim();
} }
public void setDefaultNodeLabelExpression(String queue, String exp) { public void setDefaultNodeLabelExpression(String queue, String exp) {

View File

@ -694,6 +694,53 @@ public void testCreatePreemptedContainerStatus() {
ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x"); ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x");
Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus()); Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus());
} }
@Test (timeout = 30000)
public void testNormalizeNodeLabelExpression()
throws IOException {
// mock queue and scheduler
YarnScheduler scheduler = mock(YarnScheduler.class);
Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
QueueInfo queueInfo = mock(QueueInfo.class);
when(queueInfo.getQueueName()).thenReturn("queue");
when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels);
when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x ");
when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean()))
.thenReturn(queueInfo);
Resource maxResource = Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
// queue has labels, success cases
try {
// set queue accessible node labels to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
rmContext.getNodeLabelManager().addToCluserNodeLabels(
ImmutableSet.of(NodeLabel.newInstance("x"),
NodeLabel.newInstance("y")));
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler, rmContext);
Assert.assertTrue(resReq.getNodeLabelExpression().equals("x"));
resReq.setNodeLabelExpression(" y ");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler, rmContext);
Assert.assertTrue(resReq.getNodeLabelExpression().equals("y"));
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when request labels is a subset of queue labels");
} finally {
rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
Arrays.asList("x", "y"));
}
}
public static SchedulerApplication<SchedulerApplicationAttempt> public static SchedulerApplication<SchedulerApplicationAttempt>
verifyAppAddedAndRemovedFromScheduler( verifyAppAddedAndRemovedFromScheduler(

View File

@ -2930,6 +2930,27 @@ public void testApplicationHeadRoom() throws Exception {
rm.stop(); rm.stop();
} }
@Test
public void testDefaultNodeLabelExpressionQueueConfig() throws Exception {
CapacityScheduler cs = new CapacityScheduler();
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
conf.setDefaultNodeLabelExpression("root.a", " x");
conf.setDefaultNodeLabelExpression("root.b", " y ");
cs.setConf(new YarnConfiguration());
cs.setRMContext(resourceManager.getRMContext());
cs.init(conf);
cs.start();
QueueInfo queueInfoA = cs.getQueueInfo("a", true, false);
Assert.assertEquals(queueInfoA.getQueueName(), "a");
Assert.assertEquals(queueInfoA.getDefaultNodeLabelExpression(), "x");
QueueInfo queueInfoB = cs.getQueueInfo("b", true, false);
Assert.assertEquals(queueInfoB.getQueueName(), "b");
Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y");
}
private void setMaxAllocMb(Configuration conf, int maxAllocMb) { private void setMaxAllocMb(Configuration conf, int maxAllocMb) {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,