big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8111. Simplify PlacementConstraints API by removing allocationTagToIntraApp. Contributed by Weiwei Yang.

Browse Source
This commit is contained in:
Konstantinos Karanasos 2018-04-20 12:24:48 -07:00
parent 766544c0b0
commit 28e2244390
5 changed files with 28 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
View File

@ -258,15 +258,17 @@ public static TargetExpression nodePartition(
/** /**
* Constructs a target expression on an allocation tag. It is satisfied if * Constructs a target expression on an allocation tag. It is satisfied if
* there are allocations with one of the given tags. * there are allocations with one of the given tags. The default namespace
* for these tags is {@link AllocationTagNamespaceType#SELF}, this only
* checks tags within the application.
* *
* @param allocationTags the set of tags that the attribute should take * @param allocationTags the set of tags that the attribute should take
* values from * values from
* @return the resulting expression on the allocation tags * @return the resulting expression on the allocation tags
*/ */
public static TargetExpression allocationTag(String... allocationTags) { public static TargetExpression allocationTag(String... allocationTags) {
return new TargetExpression(TargetType.ALLOCATION_TAG, null, return allocationTagWithNamespace(
allocationTags); AllocationTagNamespaceType.SELF.toString(), allocationTags);
} }
/** /**
@ -282,22 +284,6 @@ public static TargetExpression allocationTagWithNamespace(String namespace,
return new TargetExpression(TargetType.ALLOCATION_TAG, return new TargetExpression(TargetType.ALLOCATION_TAG,
namespace, allocationTags); namespace, allocationTags);
} }
/**
* Constructs a target expression on an allocation tag. It is satisfied if
* there are allocations with one of the given tags. Comparing to
* {@link PlacementTargets#allocationTag(String...)}, this only checks tags
* within the application.
*
* @param allocationTags the set of tags that the attribute should take
* values from
* @return the resulting expression on the allocation tags
*/
public static TargetExpression allocationTagToIntraApp(
String... allocationTags) {
return new TargetExpression(TargetType.ALLOCATION_TAG,
AllocationTagNamespaceType.SELF.toString(), allocationTags);
}
} }
// Creation of compound constraints. // Creation of compound constraints.

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java
View File

@ -28,6 +28,7 @@
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag; import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.nodeAttribute; import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.nodeAttribute;
import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
@ -55,7 +56,8 @@ public void testNodeAffinityToTag() {
Assert.assertEquals(1, sConstraint.getTargetExpressions().size()); Assert.assertEquals(1, sConstraint.getTargetExpressions().size());
TargetExpression tExpr = TargetExpression tExpr =
sConstraint.getTargetExpressions().iterator().next(); sConstraint.getTargetExpressions().iterator().next();
Assert.assertNull(tExpr.getTargetKey()); Assert.assertEquals(AllocationTagNamespaceType.SELF.toString(),
tExpr.getTargetKey());
Assert.assertEquals(TargetType.ALLOCATION_TAG, tExpr.getTargetType()); Assert.assertEquals(TargetType.ALLOCATION_TAG, tExpr.getTargetType());
Assert.assertEquals(1, tExpr.getTargetValues().size()); Assert.assertEquals(1, tExpr.getTargetValues().size());
Assert.assertEquals("hbase-m", tExpr.getTargetValues().iterator().next()); Assert.assertEquals("hbase-m", tExpr.getTargetValues().iterator().next());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
View File

@ -486,7 +486,7 @@ public void requestContainers(long count) {
List<TargetExpression> targetExpressions = new ArrayList<>(); List<TargetExpression> targetExpressions = new ArrayList<>();
// Currently only intra-application allocation tags are supported. // Currently only intra-application allocation tags are supported.
if (!yarnServiceConstraint.getTargetTags().isEmpty()) { if (!yarnServiceConstraint.getTargetTags().isEmpty()) {
targetExpressions.add(PlacementTargets.allocationTagToIntraApp( targetExpressions.add(PlacementTargets.allocationTag(
yarnServiceConstraint.getTargetTags().toArray(new String[0]))); yarnServiceConstraint.getTargetTags().toArray(new String[0])));
} }
// Add all node attributes // Add all node attributes

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
View File

@ -336,7 +336,7 @@ public AllocateResponse allocateIntraAppAntiAffinity(
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp(tags), .allocationTag(tags),
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.nodePartition(nodePartition)).build()) .nodePartition(nodePartition)).build())
.resourceSizing(resourceSizing).build()), null); .resourceSizing(resourceSizing).build()), null);

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java
View File

@ -126,7 +126,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"), .allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -142,7 +142,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"), .allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("x")) PlacementConstraints.PlacementTargets.nodePartition("x"))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -158,7 +158,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer")).build()) .allocationTag("mapper", "reducer")).build())
.resourceSizing( .resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
.build()); .build());
@ -173,7 +173,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer")).build()) .allocationTag("mapper", "reducer")).build())
.resourceSizing( .resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
.build()); .build());
@ -189,7 +189,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer")).build()) .allocationTag("mapper", "reducer")).build())
.build(), true); .build(), true);
// Invalid (without target tags) // Invalid (without target tags)
@ -207,9 +207,9 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"), .allocationTag("mapper"),
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("reducer"), .allocationTag("reducer"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -222,9 +222,9 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"), .allocationTag("mapper"),
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp(""), .allocationTag(""),
PlacementConstraints.PlacementTargets.nodePartition("x")) PlacementConstraints.PlacementTargets.nodePartition("x"))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -237,7 +237,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetCardinality(PlacementConstraints.NODE, 1, 2, .targetCardinality(PlacementConstraints.NODE, 1, 2,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"), .allocationTag("mapper"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -250,7 +250,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetCardinality(PlacementConstraints.NODE, 0, 2, .targetCardinality(PlacementConstraints.NODE, 0, 2,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"), .allocationTag("mapper"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -263,7 +263,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.RACK, .targetNotIn(PlacementConstraints.RACK,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"), .allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -276,7 +276,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"), .allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -292,7 +292,7 @@ public void testSchedulingRequestUpdate() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"), .allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -321,7 +321,7 @@ public void testSchedulingRequestUpdate() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetCardinality(PlacementConstraints.NODE, 0, 1, .targetCardinality(PlacementConstraints.NODE, 0, 1,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"), .allocationTag("mapper"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -338,7 +338,7 @@ public void testSchedulingRequestUpdate() {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"), .allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -358,7 +358,7 @@ public void testFunctionality() throws InvalidAllocationTagsQueryException {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"), .allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("")) PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -379,7 +379,7 @@ public void testFunctionality() throws InvalidAllocationTagsQueryException {
.placementConstraintExpression(PlacementConstraints .placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE, .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"), .allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("x")) PlacementConstraints.PlacementTargets.nodePartition("x"))
.build()).resourceSizing( .build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1))) ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))