big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9209. When nodePartition is not set in Placement Constraints, containers are allocated only in default partition. Contributed by Tarun Parimi.

Browse Source
This commit is contained in:
Weiwei Yang 2019-06-21 17:40:55 +08:00
parent b95a58e231
commit 83dcb9d87e
3 changed files with 22 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
View File

@ -795,4 +795,18 @@ public boolean precheckNode(SchedulerRequestKey schedulerKey,
public Map<String, String> getApplicationSchedulingEnvs() { public Map<String, String> getApplicationSchedulingEnvs() {
return applicationSchedulingEnvs; return applicationSchedulingEnvs;
} }
/**
* Get the defaultNodeLabelExpression for the application's current queue.
*
* @return defaultNodeLabelExpression
*/
public String getDefaultNodeLabelExpression() {
try {
this.readLock.lock();
return queue.getDefaultNodeLabelExpression();
} finally {
this.readLock.unlock();
}
}
} }

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
View File

@ -255,7 +255,10 @@ private void validateAndSetSchedulingRequest(SchedulingRequest
// Currently only single constraint is handled. // Currently only single constraint is handled.
private String validateAndGetTargetNodePartition( private String validateAndGetTargetNodePartition(
PlacementConstraint placementConstraint) { PlacementConstraint placementConstraint) {
String nodePartition = RMNodeLabelsManager.NO_LABEL; String defaultNodeLabelExpression =
appSchedulingInfo.getDefaultNodeLabelExpression();
String nodePartition = defaultNodeLabelExpression == null ?
RMNodeLabelsManager.NO_LABEL : defaultNodeLabelExpression;
if (placementConstraint != null && if (placementConstraint != null &&
placementConstraint.getConstraintExpr() != null) { placementConstraint.getConstraintExpr() != null) {
PlacementConstraint.AbstractConstraint ac = PlacementConstraint.AbstractConstraint ac =

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java
View File

@ -67,7 +67,7 @@ public void setup() throws Exception {
TestUtils.getMockApplicationId(1)); TestUtils.getMockApplicationId(1));
when(appSchedulingInfo.getApplicationAttemptId()).thenReturn( when(appSchedulingInfo.getApplicationAttemptId()).thenReturn(
TestUtils.getMockApplicationAttemptId(1, 1)); TestUtils.getMockApplicationAttemptId(1, 1));
when(appSchedulingInfo.getDefaultNodeLabelExpression()).thenReturn("y");
// stub RMContext // stub RMContext
rmContext = TestUtils.getMockRMContext(); rmContext = TestUtils.getMockRMContext();
@ -153,7 +153,8 @@ public void testSchedulingRequestValidation() {
.resourceSizing( .resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
.build()); .build());
Assert.assertEquals("", allocator.getTargetNodePartition()); // Node partition is unspecified, use the default node label expression y
Assert.assertEquals("y", allocator.getTargetNodePartition());
// Valid (with application Id target) // Valid (with application Id target)
assertValidSchedulingRequest(SchedulingRequest.newBuilder().executionType( assertValidSchedulingRequest(SchedulingRequest.newBuilder().executionType(
@ -167,7 +168,7 @@ public void testSchedulingRequestValidation() {
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
.build()); .build());
// Allocation tags should not include application Id // Allocation tags should not include application Id
Assert.assertEquals("", allocator.getTargetNodePartition()); Assert.assertEquals("y", allocator.getTargetNodePartition());
// Invalid (without sizing) // Invalid (without sizing)
assertInvalidSchedulingRequest(SchedulingRequest.newBuilder().executionType( assertInvalidSchedulingRequest(SchedulingRequest.newBuilder().executionType(