diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java index c054cbc4f0..9bb17f435e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -23,6 +23,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.Iterator; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -45,6 +47,11 @@ public PlacementConstraint(AbstractConstraint constraintExpr) { this.constraintExpr = constraintExpr; } + @Override + public String toString() { + return this.constraintExpr.toString(); + } + /** * Get the constraint expression of the placement constraint. * @@ -225,6 +232,42 @@ public int hashCode() { return result; } + @Override + public String toString() { + int max = getMaxCardinality(); + int min = getMinCardinality(); + List targetExprList = getTargetExpressions().stream() + .map(TargetExpression::toString).collect(Collectors.toList()); + List targetConstraints = new ArrayList<>(); + for (String targetExpr : targetExprList) { + if (min == 0 && max == 0) { + // anti-affinity + targetConstraints.add(new StringBuilder() + .append("notin").append(",") + .append(getScope()).append(",") + .append(targetExpr) + .toString()); + } else if (min == 1 && max == Integer.MAX_VALUE) { + // affinity + targetConstraints.add(new StringBuilder() + .append("in").append(",") + .append(getScope()).append(",") + .append(targetExpr) + .toString()); + } else { + // cardinality + targetConstraints.add(new StringBuilder() + .append("cardinality").append(",") + .append(getScope()).append(",") + .append(targetExpr).append(",") + .append(min).append(",") + .append(max) + .toString()); + } + } + return String.join(":", targetConstraints); + } + @Override public T accept(Visitor visitor) { return visitor.visit(this); @@ -325,6 +368,23 @@ public boolean equals(Object o) { : that.targetValues == null; } + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + if (TargetType.ALLOCATION_TAG == this.targetType) { + // following by a comma separated tags + sb.append(String.join(",", getTargetValues())); + } else if (TargetType.NODE_ATTRIBUTE == this.targetType) { + // following by a comma separated key value pairs + if (this.getTargetValues() != null) { + String attributeName = this.getTargetKey(); + String attributeValues = String.join(":", this.getTargetValues()); + sb.append(attributeName + "=[" + attributeValues + "]"); + } + } + return sb.toString(); + } + @Override public T accept(Visitor visitor) { return visitor.visit(this); @@ -345,7 +405,16 @@ public static class TargetConstraint extends AbstractConstraint { * TargetOperator enum helps to specify type. */ enum TargetOperator { - IN, NOT_IN + IN("in"), NOT_IN("notin"); + + private String operator; + TargetOperator(String op) { + this.operator = op; + } + + String getOperator() { + return this.operator; + } } private TargetOperator op; @@ -414,6 +483,17 @@ public int hashCode() { return result; } + @Override + public String toString() { + List targetExprs = getTargetExpressions().stream().map( + targetExpression -> new StringBuilder() + .append(op.getOperator()).append(",") + .append(scope).append(",") + .append(targetExpression.toString()) + .toString()).collect(Collectors.toList()); + return String.join(":", targetExprs); + } + @Override public T accept(Visitor visitor) { return visitor.visit(this); @@ -517,6 +597,17 @@ public int hashCode() { + (allocationTags != null ? allocationTags.hashCode() : 0); return result; } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("cardinality").append(",").append(getScope()).append(","); + for (String tag : getAllocationTags()) { + sb.append(tag).append(","); + } + sb.append(minCardinality).append(",").append(maxCardinality); + return sb.toString(); + } } /** @@ -580,6 +671,22 @@ public List getChildren() { public T accept(Visitor visitor) { return visitor.visit(this); } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("and("); + Iterator it = getChildren().iterator(); + while (it.hasNext()) { + AbstractConstraint child = it.next(); + sb.append(child.toString()); + if (it.hasNext()) { + sb.append(":"); + } + } + sb.append(")"); + return sb.toString(); + } } /** @@ -606,6 +713,22 @@ public List getChildren() { public T accept(Visitor visitor) { return visitor.visit(this); } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("or("); + Iterator it = getChildren().iterator(); + while (it.hasNext()) { + AbstractConstraint child = it.next(); + sb.append(child.toString()); + if (it.hasNext()) { + sb.append(":"); + } + } + sb.append(")"); + return sb.toString(); + } } /** @@ -636,6 +759,22 @@ public List getChildren() { public T accept(Visitor visitor) { return visitor.visit(this); } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("DelayedOr("); + Iterator it = getChildren().iterator(); + while (it.hasNext()) { + TimedPlacementConstraint child = it.next(); + sb.append(child.toString()); + if (it.hasNext()) { + sb.append(","); + } + } + sb.append(")"); + return sb.toString(); + } } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java index 941f9716dc..a69571c5c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; import org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException; @@ -51,42 +52,50 @@ public class TestPlacementConstraintParser { @Test public void testTargetExpressionParser() throws PlacementConstraintParseException { + String expressionStr; ConstraintParser parser; AbstractConstraint constraint; SingleConstraint single; // Anti-affinity with single target tag - // NOTIN,NDOE,foo - parser = new TargetConstraintParser("NOTIN, NODE, foo"); + // NOTIN,NODE,foo + expressionStr = "NOTIN, NODE, foo"; + parser = new TargetConstraintParser(expressionStr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(0, single.getMaxCardinality()); + verifyConstraintToString(expressionStr, constraint); // lower cases is also valid - parser = new TargetConstraintParser("notin, node, foo"); + expressionStr = "notin, node, foo"; + parser = new TargetConstraintParser(expressionStr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(0, single.getMaxCardinality()); + verifyConstraintToString(expressionStr, constraint); // Affinity with single target tag // IN,NODE,foo - parser = new TargetConstraintParser("IN, NODE, foo"); + expressionStr = "IN, NODE, foo"; + parser = new TargetConstraintParser(expressionStr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; Assert.assertEquals("node", single.getScope()); Assert.assertEquals(1, single.getMinCardinality()); Assert.assertEquals(Integer.MAX_VALUE, single.getMaxCardinality()); + verifyConstraintToString(expressionStr, constraint); // Anti-affinity with multiple target tags // NOTIN,NDOE,foo,bar,exp - parser = new TargetConstraintParser("NOTIN, NODE, foo, bar, exp"); + expressionStr = "NOTIN, NODE, foo, bar, exp"; + parser = new TargetConstraintParser(expressionStr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; @@ -98,6 +107,7 @@ public void testTargetExpressionParser() single.getTargetExpressions().iterator().next(); Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString()); Assert.assertEquals(3, exp.getTargetValues().size()); + verifyConstraintToString(expressionStr, constraint); // Invalid OP parser = new TargetConstraintParser("XYZ, NODE, foo"); @@ -112,12 +122,14 @@ public void testTargetExpressionParser() @Test public void testCardinalityConstraintParser() throws PlacementConstraintParseException { + String expressionExpr; ConstraintParser parser; AbstractConstraint constraint; SingleConstraint single; // cardinality,NODE,foo,0,1 - parser = new CardinalityConstraintParser("cardinality, NODE, foo, 0, 1"); + expressionExpr = "cardinality, NODE, foo, 0, 1"; + parser = new CardinalityConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; @@ -130,10 +142,11 @@ public void testCardinalityConstraintParser() Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString()); Assert.assertEquals(1, exp.getTargetValues().size()); Assert.assertEquals("foo", exp.getTargetValues().iterator().next()); + verifyConstraintToString(expressionExpr, constraint); // cardinality,NODE,foo,bar,moo,0,1 - parser = new CardinalityConstraintParser( - "cardinality,RACK,foo,bar,moo,0,1"); + expressionExpr = "cardinality,RACK,foo,bar,moo,0,1"; + parser = new CardinalityConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof SingleConstraint); single = (SingleConstraint) constraint; @@ -147,6 +160,7 @@ public void testCardinalityConstraintParser() Set expectedTags = Sets.newHashSet("foo", "bar", "moo"); Assert.assertTrue(Sets.difference(expectedTags, exp.getTargetValues()) .isEmpty()); + verifyConstraintToString(expressionExpr, constraint); // Invalid scope string try { @@ -174,25 +188,29 @@ public void testCardinalityConstraintParser() @Test public void testAndConstraintParser() throws PlacementConstraintParseException { + String expressionExpr; ConstraintParser parser; AbstractConstraint constraint; And and; - parser = new ConjunctionConstraintParser( - "AND(NOTIN,NODE,foo:NOTIN,NODE,bar)"); + expressionExpr = "AND(NOTIN,NODE,foo:NOTIN,NODE,bar)"; + parser = new ConjunctionConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof And); and = (And) constraint; Assert.assertEquals(2, and.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); - parser = new ConjunctionConstraintParser( - "AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)"); + expressionExpr = "AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)"; + parser = new ConjunctionConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof And); Assert.assertEquals(2, and.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); - parser = new ConjunctionConstraintParser( - "AND(NOTIN,NODE,foo:AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))"); + expressionExpr = + "AND(NOTIN,NODE,foo:AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))"; + parser = new ConjunctionConstraintParser(expressionExpr); constraint = parser.parse(); Assert.assertTrue(constraint instanceof And); and = (And) constraint; @@ -200,6 +218,43 @@ public void testAndConstraintParser() Assert.assertTrue(and.getChildren().get(1) instanceof And); and = (And) and.getChildren().get(1); Assert.assertEquals(2, and.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); + } + + @Test + public void testOrConstraintParser() + throws PlacementConstraintParseException { + String expressionExpr; + ConstraintParser parser; + AbstractConstraint constraint; + Or or; + + expressionExpr = "OR(NOTIN,NODE,foo:NOTIN,NODE,bar)"; + parser = new ConjunctionConstraintParser(expressionExpr); + constraint = parser.parse(); + Assert.assertTrue(constraint instanceof Or); + or = (Or) constraint; + Assert.assertEquals(2, or.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); + + expressionExpr = "OR(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)"; + parser = new ConjunctionConstraintParser(expressionExpr); + constraint = parser.parse(); + Assert.assertTrue(constraint instanceof Or); + Assert.assertEquals(2, or.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); + + expressionExpr = + "OR(NOTIN,NODE,foo:OR(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))"; + parser = new ConjunctionConstraintParser(expressionExpr); + constraint = parser.parse(); + Assert.assertTrue(constraint instanceof Or); + or = (Or) constraint; + Assert.assertTrue(or.getChildren().get(0) instanceof SingleConstraint); + Assert.assertTrue(or.getChildren().get(1) instanceof Or); + or = (Or) or.getChildren().get(1); + Assert.assertEquals(2, or.getChildren().size()); + verifyConstraintToString(expressionExpr, constraint); } @Test @@ -369,4 +424,23 @@ public void testParsePlacementSpec() expectedPc2 = targetNotIn("node", allocationTag("foo")).build(); Assert.assertEquals(expectedPc2, actualPc2); } + + // We verify the toString result by parsing it again + // instead of raw string comparing. This is because internally + // we are not storing tags strictly to its original order, so + // the toString result might have different ordering with the + // input expression. + private void verifyConstraintToString(String inputExpr, + AbstractConstraint constraint) { + String constrainExpr = constraint.toString(); + System.out.println("Input: " + inputExpr + .toLowerCase().replaceAll(" ", "")); + System.out.println("ToString: " + constrainExpr); + try { + PlacementConstraintParser.parseExpression(constrainExpr); + } catch (PlacementConstraintParseException e) { + Assert.fail("The parser is unable to parse the expression: " + + constrainExpr + ", caused by: " + e.getMessage()); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java index aa92d7a373..557b82fe25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java @@ -65,6 +65,10 @@ public void testTargetConstraint() { SingleConstraint single = (SingleConstraint) sConstraintExpr; TargetConstraint target = (TargetConstraint) tConstraintExpr; + + // Make sure the expression string is consistent + // before and after transforming + Assert.assertEquals(single.toString(), target.toString()); Assert.assertEquals(single.getScope(), target.getScope()); Assert.assertEquals(TargetOperator.IN, target.getOp()); Assert.assertEquals(single.getTargetExpressions(), @@ -101,6 +105,9 @@ public void testCardinalityConstraint() { Assert.assertTrue(sConstraintExpr instanceof SingleConstraint); SingleConstraint single = (SingleConstraint) sConstraintExpr; + // Make sure the consistent expression string is consistent + // before and after transforming + Assert.assertEquals(single.toString(), cardinality.toString()); Assert.assertEquals(cardinality.getScope(), single.getScope()); Assert.assertEquals(cardinality.getMinCardinality(), single.getMinCardinality());