big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8111. Simplify PlacementConstraints API by removing allocationTagToIntraApp. Contributed by Weiwei Yang.

Browse Source
This commit is contained in:
Konstantinos Karanasos 2018-04-20 12:24:48 -07:00
parent 766544c0b0
commit 28e2244390
5 changed files with 28 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
View File

@ -258,15 +258,17 @@ public static TargetExpression nodePartition(
/**
* Constructs a target expression on an allocation tag. It is satisfied if
* there are allocations with one of the given tags.
* there are allocations with one of the given tags. The default namespace
* for these tags is {@link AllocationTagNamespaceType#SELF}, this only
* checks tags within the application.
*
* @param allocationTags the set of tags that the attribute should take
* values from
* @return the resulting expression on the allocation tags
*/
public static TargetExpression allocationTag(String... allocationTags) {
return new TargetExpression(TargetType.ALLOCATION_TAG, null,
allocationTags);
return allocationTagWithNamespace(
AllocationTagNamespaceType.SELF.toString(), allocationTags);
}
/**
@ -282,22 +284,6 @@ public static TargetExpression allocationTagWithNamespace(String namespace,
return new TargetExpression(TargetType.ALLOCATION_TAG,
namespace, allocationTags);
}
/**
* Constructs a target expression on an allocation tag. It is satisfied if
* there are allocations with one of the given tags. Comparing to
* {@link PlacementTargets#allocationTag(String...)}, this only checks tags
* within the application.
*
* @param allocationTags the set of tags that the attribute should take
* values from
* @return the resulting expression on the allocation tags
*/
public static TargetExpression allocationTagToIntraApp(
String... allocationTags) {
return new TargetExpression(TargetType.ALLOCATION_TAG,
AllocationTagNamespaceType.SELF.toString(), allocationTags);
}
}
// Creation of compound constraints.

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java
View File

@ -28,6 +28,7 @@
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.nodeAttribute;
import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
@ -55,7 +56,8 @@ public void testNodeAffinityToTag() {
Assert.assertEquals(1, sConstraint.getTargetExpressions().size());
TargetExpression tExpr =
sConstraint.getTargetExpressions().iterator().next();
Assert.assertNull(tExpr.getTargetKey());
Assert.assertEquals(AllocationTagNamespaceType.SELF.toString(),
tExpr.getTargetKey());
Assert.assertEquals(TargetType.ALLOCATION_TAG, tExpr.getTargetType());
Assert.assertEquals(1, tExpr.getTargetValues().size());
Assert.assertEquals("hbase-m", tExpr.getTargetValues().iterator().next());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
View File

@ -486,7 +486,7 @@ public void requestContainers(long count) {
List<TargetExpression> targetExpressions = new ArrayList<>();
// Currently only intra-application allocation tags are supported.
if (!yarnServiceConstraint.getTargetTags().isEmpty()) {
targetExpressions.add(PlacementTargets.allocationTagToIntraApp(
targetExpressions.add(PlacementTargets.allocationTag(
yarnServiceConstraint.getTargetTags().toArray(new String[0])));
}
// Add all node attributes

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
View File

@ -336,7 +336,7 @@ public AllocateResponse allocateIntraAppAntiAffinity(
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp(tags),
.allocationTag(tags),
PlacementConstraints.PlacementTargets
.nodePartition(nodePartition)).build())
.resourceSizing(resourceSizing).build()), null);

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java
View File

@ -126,7 +126,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
.allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -142,7 +142,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
.allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("x"))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -158,7 +158,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer")).build())
.allocationTag("mapper", "reducer")).build())
.resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
.build());
@ -173,7 +173,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer")).build())
.allocationTag("mapper", "reducer")).build())
.resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
.build());
@ -189,7 +189,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer")).build())
.allocationTag("mapper", "reducer")).build())
.build(), true);
// Invalid (without target tags)
@ -207,9 +207,9 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"),
.allocationTag("mapper"),
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("reducer"),
.allocationTag("reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -222,9 +222,9 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"),
.allocationTag("mapper"),
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp(""),
.allocationTag(""),
PlacementConstraints.PlacementTargets.nodePartition("x"))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -237,7 +237,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetCardinality(PlacementConstraints.NODE, 1, 2,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"),
.allocationTag("mapper"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -250,7 +250,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetCardinality(PlacementConstraints.NODE, 0, 2,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"),
.allocationTag("mapper"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -263,7 +263,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.RACK,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
.allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -276,7 +276,7 @@ public void testSchedulingRequestValidation() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
.allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -292,7 +292,7 @@ public void testSchedulingRequestUpdate() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
.allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -321,7 +321,7 @@ public void testSchedulingRequestUpdate() {
.placementConstraintExpression(PlacementConstraints
.targetCardinality(PlacementConstraints.NODE, 0, 1,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"),
.allocationTag("mapper"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -338,7 +338,7 @@ public void testSchedulingRequestUpdate() {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
.allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -358,7 +358,7 @@ public void testFunctionality() throws InvalidAllocationTagsQueryException {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
.allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))
@ -379,7 +379,7 @@ public void testFunctionality() throws InvalidAllocationTagsQueryException {
.placementConstraintExpression(PlacementConstraints
.targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
.allocationTag("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("x"))
.build()).resourceSizing(
ResourceSizing.newInstance(1, Resource.newInstance(1024, 1)))