big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7921. Transform a PlacementConstraint to a string expression. Contributed by Weiwei Yang.

Browse Source
This commit is contained in:
Konstantinos Karanasos 2018-02-26 12:15:16 -08:00
parent 451265a83d
commit e85188101c
3 changed files with 235 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
View File

@ -23,6 +23,8 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@ -45,6 +47,11 @@ public PlacementConstraint(AbstractConstraint constraintExpr) {
this.constraintExpr = constraintExpr;
}
@Override
public String toString() {
return this.constraintExpr.toString();
}
/**
* Get the constraint expression of the placement constraint.
*
@ -225,6 +232,42 @@ public int hashCode() {
return result;
}
@Override
public String toString() {
int max = getMaxCardinality();
int min = getMinCardinality();
List<String> targetExprList = getTargetExpressions().stream()
.map(TargetExpression::toString).collect(Collectors.toList());
List<String> targetConstraints = new ArrayList<>();
for (String targetExpr : targetExprList) {
if (min == 0 && max == 0) {
// anti-affinity
targetConstraints.add(new StringBuilder()
.append("notin").append(",")
.append(getScope()).append(",")
.append(targetExpr)
.toString());
} else if (min == 1 && max == Integer.MAX_VALUE) {
// affinity
targetConstraints.add(new StringBuilder()
.append("in").append(",")
.append(getScope()).append(",")
.append(targetExpr)
.toString());
} else {
// cardinality
targetConstraints.add(new StringBuilder()
.append("cardinality").append(",")
.append(getScope()).append(",")
.append(targetExpr).append(",")
.append(min).append(",")
.append(max)
.toString());
}
}
return String.join(":", targetConstraints);
}
@Override
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
@ -325,6 +368,23 @@ public boolean equals(Object o) {
: that.targetValues == null;
}
@Override
public String toString() {
StringBuffer sb = new StringBuffer();
if (TargetType.ALLOCATION_TAG == this.targetType) {
// following by a comma separated tags
sb.append(String.join(",", getTargetValues()));
} else if (TargetType.NODE_ATTRIBUTE == this.targetType) {
// following by a comma separated key value pairs
if (this.getTargetValues() != null) {
String attributeName = this.getTargetKey();
String attributeValues = String.join(":", this.getTargetValues());
sb.append(attributeName + "=[" + attributeValues + "]");
}
}
return sb.toString();
}
@Override
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
@ -345,7 +405,16 @@ public static class TargetConstraint extends AbstractConstraint {
* TargetOperator enum helps to specify type.
*/
enum TargetOperator {
IN, NOT_IN
IN("in"), NOT_IN("notin");
private String operator;
TargetOperator(String op) {
this.operator = op;
}
String getOperator() {
return this.operator;
}
}
private TargetOperator op;
@ -414,6 +483,17 @@ public int hashCode() {
return result;
}
@Override
public String toString() {
List<String> targetExprs = getTargetExpressions().stream().map(
targetExpression -> new StringBuilder()
.append(op.getOperator()).append(",")
.append(scope).append(",")
.append(targetExpression.toString())
.toString()).collect(Collectors.toList());
return String.join(":", targetExprs);
}
@Override
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
@ -517,6 +597,17 @@ public int hashCode() {
+ (allocationTags != null ? allocationTags.hashCode() : 0);
return result;
}
@Override
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("cardinality").append(",").append(getScope()).append(",");
for (String tag : getAllocationTags()) {
sb.append(tag).append(",");
}
sb.append(minCardinality).append(",").append(maxCardinality);
return sb.toString();
}
}
/**
@ -580,6 +671,22 @@ public List<AbstractConstraint> getChildren() {
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
@Override
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("and(");
Iterator<AbstractConstraint> it = getChildren().iterator();
while (it.hasNext()) {
AbstractConstraint child = it.next();
sb.append(child.toString());
if (it.hasNext()) {
sb.append(":");
}
}
sb.append(")");
return sb.toString();
}
}
/**
@ -606,6 +713,22 @@ public List<AbstractConstraint> getChildren() {
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
@Override
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("or(");
Iterator<AbstractConstraint> it = getChildren().iterator();
while (it.hasNext()) {
AbstractConstraint child = it.next();
sb.append(child.toString());
if (it.hasNext()) {
sb.append(":");
}
}
sb.append(")");
return sb.toString();
}
}
/**
@ -636,6 +759,22 @@ public List<TimedPlacementConstraint> getChildren() {
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
@Override
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("DelayedOr(");
Iterator<TimedPlacementConstraint> it = getChildren().iterator();
while (it.hasNext()) {
TimedPlacementConstraint child = it.next();
sb.append(child.toString());
if (it.hasNext()) {
sb.append(",");
}
}
sb.append(")");
return sb.toString();
}
}
/**

102
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java
View File

@ -24,6 +24,7 @@
import java.util.Set;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
import org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException;
@ -51,42 +52,50 @@ public class TestPlacementConstraintParser {
@Test
public void testTargetExpressionParser()
throws PlacementConstraintParseException {
String expressionStr;
ConstraintParser parser;
AbstractConstraint constraint;
SingleConstraint single;
// Anti-affinity with single target tag
// NOTIN,NDOE,foo
parser = new TargetConstraintParser("NOTIN, NODE, foo");
// NOTIN,NODE,foo
expressionStr = "NOTIN, NODE, foo";
parser = new TargetConstraintParser(expressionStr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
Assert.assertEquals("node", single.getScope());
Assert.assertEquals(0, single.getMinCardinality());
Assert.assertEquals(0, single.getMaxCardinality());
verifyConstraintToString(expressionStr, constraint);
// lower cases is also valid
parser = new TargetConstraintParser("notin, node, foo");
expressionStr = "notin, node, foo";
parser = new TargetConstraintParser(expressionStr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
Assert.assertEquals("node", single.getScope());
Assert.assertEquals(0, single.getMinCardinality());
Assert.assertEquals(0, single.getMaxCardinality());
verifyConstraintToString(expressionStr, constraint);
// Affinity with single target tag
// IN,NODE,foo
parser = new TargetConstraintParser("IN, NODE, foo");
expressionStr = "IN, NODE, foo";
parser = new TargetConstraintParser(expressionStr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
Assert.assertEquals("node", single.getScope());
Assert.assertEquals(1, single.getMinCardinality());
Assert.assertEquals(Integer.MAX_VALUE, single.getMaxCardinality());
verifyConstraintToString(expressionStr, constraint);
// Anti-affinity with multiple target tags
// NOTIN,NDOE,foo,bar,exp
parser = new TargetConstraintParser("NOTIN, NODE, foo, bar, exp");
expressionStr = "NOTIN, NODE, foo, bar, exp";
parser = new TargetConstraintParser(expressionStr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
@ -98,6 +107,7 @@ public void testTargetExpressionParser()
single.getTargetExpressions().iterator().next();
Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString());
Assert.assertEquals(3, exp.getTargetValues().size());
verifyConstraintToString(expressionStr, constraint);
// Invalid OP
parser = new TargetConstraintParser("XYZ, NODE, foo");
@ -112,12 +122,14 @@ public void testTargetExpressionParser()
@Test
public void testCardinalityConstraintParser()
throws PlacementConstraintParseException {
String expressionExpr;
ConstraintParser parser;
AbstractConstraint constraint;
SingleConstraint single;
// cardinality,NODE,foo,0,1
parser = new CardinalityConstraintParser("cardinality, NODE, foo, 0, 1");
expressionExpr = "cardinality, NODE, foo, 0, 1";
parser = new CardinalityConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
@ -130,10 +142,11 @@ public void testCardinalityConstraintParser()
Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString());
Assert.assertEquals(1, exp.getTargetValues().size());
Assert.assertEquals("foo", exp.getTargetValues().iterator().next());
verifyConstraintToString(expressionExpr, constraint);
// cardinality,NODE,foo,bar,moo,0,1
parser = new CardinalityConstraintParser(
"cardinality,RACK,foo,bar,moo,0,1");
expressionExpr = "cardinality,RACK,foo,bar,moo,0,1";
parser = new CardinalityConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
@ -147,6 +160,7 @@ public void testCardinalityConstraintParser()
Set<String> expectedTags = Sets.newHashSet("foo", "bar", "moo");
Assert.assertTrue(Sets.difference(expectedTags, exp.getTargetValues())
.isEmpty());
verifyConstraintToString(expressionExpr, constraint);
// Invalid scope string
try {
@ -174,25 +188,29 @@ public void testCardinalityConstraintParser()
@Test
public void testAndConstraintParser()
throws PlacementConstraintParseException {
String expressionExpr;
ConstraintParser parser;
AbstractConstraint constraint;
And and;
parser = new ConjunctionConstraintParser(
"AND(NOTIN,NODE,foo:NOTIN,NODE,bar)");
expressionExpr = "AND(NOTIN,NODE,foo:NOTIN,NODE,bar)";
parser = new ConjunctionConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof And);
and = (And) constraint;
Assert.assertEquals(2, and.getChildren().size());
verifyConstraintToString(expressionExpr, constraint);
parser = new ConjunctionConstraintParser(
"AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)");
expressionExpr = "AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)";
parser = new ConjunctionConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof And);
Assert.assertEquals(2, and.getChildren().size());
verifyConstraintToString(expressionExpr, constraint);
parser = new ConjunctionConstraintParser(
"AND(NOTIN,NODE,foo:AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))");
expressionExpr =
"AND(NOTIN,NODE,foo:AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))";
parser = new ConjunctionConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof And);
and = (And) constraint;
@ -200,6 +218,43 @@ public void testAndConstraintParser()
Assert.assertTrue(and.getChildren().get(1) instanceof And);
and = (And) and.getChildren().get(1);
Assert.assertEquals(2, and.getChildren().size());
verifyConstraintToString(expressionExpr, constraint);
}
@Test
public void testOrConstraintParser()
throws PlacementConstraintParseException {
String expressionExpr;
ConstraintParser parser;
AbstractConstraint constraint;
Or or;
expressionExpr = "OR(NOTIN,NODE,foo:NOTIN,NODE,bar)";
parser = new ConjunctionConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof Or);
or = (Or) constraint;
Assert.assertEquals(2, or.getChildren().size());
verifyConstraintToString(expressionExpr, constraint);
expressionExpr = "OR(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)";
parser = new ConjunctionConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof Or);
Assert.assertEquals(2, or.getChildren().size());
verifyConstraintToString(expressionExpr, constraint);
expressionExpr =
"OR(NOTIN,NODE,foo:OR(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))";
parser = new ConjunctionConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof Or);
or = (Or) constraint;
Assert.assertTrue(or.getChildren().get(0) instanceof SingleConstraint);
Assert.assertTrue(or.getChildren().get(1) instanceof Or);
or = (Or) or.getChildren().get(1);
Assert.assertEquals(2, or.getChildren().size());
verifyConstraintToString(expressionExpr, constraint);
}
@Test
@ -369,4 +424,23 @@ public void testParsePlacementSpec()
expectedPc2 = targetNotIn("node", allocationTag("foo")).build();
Assert.assertEquals(expectedPc2, actualPc2);
}
// We verify the toString result by parsing it again
// instead of raw string comparing. This is because internally
// we are not storing tags strictly to its original order, so
// the toString result might have different ordering with the
// input expression.
private void verifyConstraintToString(String inputExpr,
AbstractConstraint constraint) {
String constrainExpr = constraint.toString();
System.out.println("Input: " + inputExpr
.toLowerCase().replaceAll(" ", ""));
System.out.println("ToString: " + constrainExpr);
try {
PlacementConstraintParser.parseExpression(constrainExpr);
} catch (PlacementConstraintParseException e) {
Assert.fail("The parser is unable to parse the expression: "
+ constrainExpr + ", caused by: " + e.getMessage());
}
}
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
View File

@ -65,6 +65,10 @@ public void testTargetConstraint() {
SingleConstraint single = (SingleConstraint) sConstraintExpr;
TargetConstraint target = (TargetConstraint) tConstraintExpr;
// Make sure the expression string is consistent
// before and after transforming
Assert.assertEquals(single.toString(), target.toString());
Assert.assertEquals(single.getScope(), target.getScope());
Assert.assertEquals(TargetOperator.IN, target.getOp());
Assert.assertEquals(single.getTargetExpressions(),
@ -101,6 +105,9 @@ public void testCardinalityConstraint() {
Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
SingleConstraint single = (SingleConstraint) sConstraintExpr;
// Make sure the consistent expression string is consistent
// before and after transforming
Assert.assertEquals(single.toString(), cardinality.toString());
Assert.assertEquals(cardinality.getScope(), single.getScope());
Assert.assertEquals(cardinality.getMinCardinality(),
single.getMinCardinality());