big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8350. NPE in service AM related to placement policy. Contributed by Gour Saha

Browse Source
This commit is contained in:
Billie Rinaldi 2018-05-30 13:19:13 -07:00
parent 96eefcc84a
commit 778a4a24be
4 changed files with 130 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
View File

@ -694,62 +694,66 @@ public void requestContainers(long count) {
// composite constraints then this AND-ed composite constraint is not
// used.
PlacementConstraint finalConstraint = null;
for (org.apache.hadoop.yarn.service.api.records.PlacementConstraint
yarnServiceConstraint : placementPolicy.getConstraints()) {
List<TargetExpression> targetExpressions = new ArrayList<>();
// Currently only intra-application allocation tags are supported.
if (!yarnServiceConstraint.getTargetTags().isEmpty()) {
targetExpressions.add(PlacementTargets.allocationTag(
yarnServiceConstraint.getTargetTags().toArray(new String[0])));
if (placementPolicy != null) {
for (org.apache.hadoop.yarn.service.api.records.PlacementConstraint
yarnServiceConstraint : placementPolicy.getConstraints()) {
List<TargetExpression> targetExpressions = new ArrayList<>();
// Currently only intra-application allocation tags are supported.
if (!yarnServiceConstraint.getTargetTags().isEmpty()) {
targetExpressions.add(PlacementTargets.allocationTag(
yarnServiceConstraint.getTargetTags().toArray(new String[0])));
}
// Add all node attributes
for (Map.Entry<String, List<String>> attribute : yarnServiceConstraint
.getNodeAttributes().entrySet()) {
targetExpressions
.add(PlacementTargets.nodeAttribute(attribute.getKey(),
attribute.getValue().toArray(new String[0])));
}
// Add all node partitions
if (!yarnServiceConstraint.getNodePartitions().isEmpty()) {
targetExpressions
.add(PlacementTargets.nodePartition(yarnServiceConstraint
.getNodePartitions().toArray(new String[0])));
}
PlacementConstraint constraint = null;
switch (yarnServiceConstraint.getType()) {
case AFFINITY:
constraint = PlacementConstraints
.targetIn(yarnServiceConstraint.getScope().getValue(),
targetExpressions.toArray(new TargetExpression[0]))
.build();
break;
case ANTI_AFFINITY:
constraint = PlacementConstraints
.targetNotIn(yarnServiceConstraint.getScope().getValue(),
targetExpressions.toArray(new TargetExpression[0]))
.build();
break;
case AFFINITY_WITH_CARDINALITY:
constraint = PlacementConstraints.targetCardinality(
yarnServiceConstraint.getScope().name().toLowerCase(),
yarnServiceConstraint.getMinCardinality() == null ? 0
: yarnServiceConstraint.getMinCardinality().intValue(),
yarnServiceConstraint.getMaxCardinality() == null
? Integer.MAX_VALUE
: yarnServiceConstraint.getMaxCardinality().intValue(),
targetExpressions.toArray(new TargetExpression[0])).build();
break;
}
// The default AND-ed final composite constraint
if (finalConstraint != null) {
finalConstraint = PlacementConstraints
.and(constraint.getConstraintExpr(),
finalConstraint.getConstraintExpr())
.build();
} else {
finalConstraint = constraint;
}
LOG.debug("[COMPONENT {}] Placement constraint: {}",
componentSpec.getName(),
constraint.getConstraintExpr().toString());
}
// Add all node attributes
for (Map.Entry<String, List<String>> attribute : yarnServiceConstraint
.getNodeAttributes().entrySet()) {
targetExpressions.add(PlacementTargets.nodeAttribute(
attribute.getKey(), attribute.getValue().toArray(new String[0])));
}
// Add all node partitions
if (!yarnServiceConstraint.getNodePartitions().isEmpty()) {
targetExpressions
.add(PlacementTargets.nodePartition(yarnServiceConstraint
.getNodePartitions().toArray(new String[0])));
}
PlacementConstraint constraint = null;
switch (yarnServiceConstraint.getType()) {
case AFFINITY:
constraint = PlacementConstraints
.targetIn(yarnServiceConstraint.getScope().getValue(),
targetExpressions.toArray(new TargetExpression[0]))
.build();
break;
case ANTI_AFFINITY:
constraint = PlacementConstraints
.targetNotIn(yarnServiceConstraint.getScope().getValue(),
targetExpressions.toArray(new TargetExpression[0]))
.build();
break;
case AFFINITY_WITH_CARDINALITY:
constraint = PlacementConstraints.targetCardinality(
yarnServiceConstraint.getScope().name().toLowerCase(),
yarnServiceConstraint.getMinCardinality() == null ? 0
: yarnServiceConstraint.getMinCardinality().intValue(),
yarnServiceConstraint.getMaxCardinality() == null
? Integer.MAX_VALUE
: yarnServiceConstraint.getMaxCardinality().intValue(),
targetExpressions.toArray(new TargetExpression[0])).build();
break;
}
// The default AND-ed final composite constraint
if (finalConstraint != null) {
finalConstraint = PlacementConstraints
.and(constraint.getConstraintExpr(),
finalConstraint.getConstraintExpr())
.build();
} else {
finalConstraint = constraint;
}
LOG.debug("[COMPONENT {}] Placement constraint: {}",
componentSpec.getName(), constraint.getConstraintExpr().toString());
}
ResourceSizing resourceSizing = ResourceSizing.newInstance((int) count,
resource);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
View File

@ -91,6 +91,14 @@ public interface RestApiErrorMessages {
String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
+ " component level, needs corresponding values set at service level";
// Note: %sin is not a typo. Constraint name is optional so the error messages
// below handle that scenario by adding a space if name is specified.
String ERROR_PLACEMENT_POLICY_CONSTRAINT_TYPE_NULL = "Type not specified "
+ "for constraint %sin placement policy of component %s.";
String ERROR_PLACEMENT_POLICY_CONSTRAINT_SCOPE_NULL = "Scope not specified "
+ "for constraint %sin placement policy of component %s.";
String ERROR_PLACEMENT_POLICY_CONSTRAINT_TAGS_NULL = "Tag(s) not specified "
+ "for constraint %sin placement policy of component %s.";
String ERROR_PLACEMENT_POLICY_TAG_NAME_NOT_SAME = "Invalid target tag %s "
+ "specified in placement policy of component %s. For now, target tags "
+ "support self reference only. Specifying anything other than its "

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
View File

@ -40,6 +40,7 @@
import org.apache.hadoop.yarn.service.api.records.Configuration;
import org.apache.hadoop.yarn.service.api.records.KerberosPrincipal;
import org.apache.hadoop.yarn.service.api.records.PlacementConstraint;
import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.conf.RestApiConstants;
@ -314,9 +315,28 @@ public static void validateNameFormat(String name,
private static void validatePlacementPolicy(List<Component> components,
Set<String> componentNames) {
for (Component comp : components) {
if (comp.getPlacementPolicy() != null) {
for (PlacementConstraint constraint : comp.getPlacementPolicy()
PlacementPolicy placementPolicy = comp.getPlacementPolicy();
if (placementPolicy != null) {
for (PlacementConstraint constraint : placementPolicy
.getConstraints()) {
if (constraint.getType() == null) {
throw new IllegalArgumentException(String.format(
RestApiErrorMessages.ERROR_PLACEMENT_POLICY_CONSTRAINT_TYPE_NULL,
constraint.getName() == null ? "" : constraint.getName() + " ",
comp.getName()));
}
if (constraint.getScope() == null) {
throw new IllegalArgumentException(String.format(
RestApiErrorMessages.ERROR_PLACEMENT_POLICY_CONSTRAINT_SCOPE_NULL,
constraint.getName() == null ? "" : constraint.getName() + " ",
comp.getName()));
}
if (constraint.getTargetTags().isEmpty()) {
throw new IllegalArgumentException(String.format(
RestApiErrorMessages.ERROR_PLACEMENT_POLICY_CONSTRAINT_TAGS_NULL,
constraint.getName() == null ? "" : constraint.getName() + " ",
comp.getName()));
}
for (String targetTag : constraint.getTargetTags()) {
if (!comp.getName().equals(targetTag)) {
throw new IllegalArgumentException(String.format(

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
View File

@ -25,6 +25,8 @@
import org.apache.hadoop.yarn.service.api.records.KerberosPrincipal;
import org.apache.hadoop.yarn.service.api.records.PlacementConstraint;
import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
import org.apache.hadoop.yarn.service.api.records.PlacementScope;
import org.apache.hadoop.yarn.service.api.records.PlacementType;
import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
@ -503,13 +505,48 @@ public void testPlacementPolicy() throws IOException {
PlacementPolicy pp = new PlacementPolicy();
PlacementConstraint pc = new PlacementConstraint();
pc.setName("CA1");
pc.setTargetTags(Collections.singletonList("comp-invalid"));
pp.setConstraints(Collections.singletonList(pc));
comp.setPlacementPolicy(pp);
try {
ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "service with empty placement");
Assert.fail(EXCEPTION_PREFIX + "constraint with no type");
} catch (IllegalArgumentException e) {
assertEquals(String.format(
RestApiErrorMessages.ERROR_PLACEMENT_POLICY_CONSTRAINT_TYPE_NULL,
"CA1 ", "comp-a"), e.getMessage());
}
// Set the type
pc.setType(PlacementType.ANTI_AFFINITY);
try {
ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "constraint with no scope");
} catch (IllegalArgumentException e) {
assertEquals(String.format(
RestApiErrorMessages.ERROR_PLACEMENT_POLICY_CONSTRAINT_SCOPE_NULL,
"CA1 ", "comp-a"), e.getMessage());
}
// Set the scope
pc.setScope(PlacementScope.NODE);
try {
ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "constraint with no tag(s)");
} catch (IllegalArgumentException e) {
assertEquals(String.format(
RestApiErrorMessages.ERROR_PLACEMENT_POLICY_CONSTRAINT_TAGS_NULL,
"CA1 ", "comp-a"), e.getMessage());
}
// Set a target tag - but an invalid one
pc.setTargetTags(Collections.singletonList("comp-invalid"));
try {
ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "constraint with invalid tag name");
} catch (IllegalArgumentException e) {
assertEquals(
String.format(
@ -518,9 +555,10 @@ public void testPlacementPolicy() throws IOException {
e.getMessage());
}
// Set valid target tags now
pc.setTargetTags(Collections.singletonList("comp-a"));
// now it should succeed
// Finally it should succeed
try {
ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {