big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8987. Usability improvements node-attributes CLI. Contributed by Bibin A Chundatt.

Browse Source
This commit is contained in:
Weiwei Yang 2018-11-12 18:18:23 +08:00
parent 9c32b50d61
commit c741109522
3 changed files with 53 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
View File

@ -205,6 +205,9 @@ public int run(String[] args) throws Exception {
// print admin command detail
printUsage(true, handler);
return exitCode;
} catch (YarnException e) {
errOut.println(e.toString());
return exitCode;
} catch (Exception e) {
errOut.println(e.toString());
printUsage(true, handler);

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
View File

@ -993,6 +993,7 @@ public NodesToAttributesMappingResponse mapAttributesToNodes(
nodeAttributesManager.addNodeAttributes(nodeAttributeMapping);
break;
case REMOVE:
validateAttributesExists(nodesToAttributes);
nodeAttributesManager.removeNodeAttributes(nodeAttributeMapping);
break;
case REPLACE:
@ -1013,6 +1014,27 @@ public NodesToAttributesMappingResponse mapAttributesToNodes(
.newRecordInstance(NodesToAttributesMappingResponse.class);
}
private void validateAttributesExists(
List<NodeToAttributes> nodesToAttributes) throws IOException {
NodeAttributesManager nodeAttributesManager =
rm.getRMContext().getNodeAttributesManager();
for (NodeToAttributes nodeToAttrs : nodesToAttributes) {
String hostname = nodeToAttrs.getNode();
if (hostname == null) {
continue;
}
Set<NodeAttribute> attrs =
nodeAttributesManager.getAttributesForNode(hostname).keySet();
List<NodeAttribute> attributes = nodeToAttrs.getNodeAttributes();
for (NodeAttribute nodeAttr : attributes) {
if (!attrs.contains(nodeAttr)) {
throw new IOException("Node attribute [" + nodeAttr.getAttributeKey()
+ "] doesn't exist on node " + nodeToAttrs.getNode());
}
}
}
}
/**
* @param nodesToAttributesMapping input to be validated
* @param failOnUnknownNodes indicates to fail if the nodes are not available.

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
View File

@ -1585,6 +1585,16 @@ public void testMapAttributesToNodes() throws Exception, YarnException {
Mockito.verify(spiedAttributesManager, Mockito.times(1))
.addNodeAttributes(Mockito.anyMap());
// add attributes for later removals
request =
NodesToAttributesMappingRequest
.newInstance(AttributeMappingOperationType.ADD,
ImmutableList.of(NodeToAttributes.newInstance("host4",
ImmutableList.of(NodeAttribute.newInstance(
NodeAttribute.PREFIX_CENTRALIZED, "x",
NodeAttributeType.STRING, "dfasdf")))),
true);
rm.adminService.mapAttributesToNodes(request);
request =
NodesToAttributesMappingRequest
.newInstance(AttributeMappingOperationType.REMOVE,
@ -1601,6 +1611,24 @@ public void testMapAttributesToNodes() throws Exception, YarnException {
Mockito.verify(spiedAttributesManager, Mockito.times(1))
.removeNodeAttributes(Mockito.anyMap());
// Assert node to attributes mappings are empty.
Assert.assertTrue("Attributes of host4 should be empty",
rm.getRMContext().getNodeAttributesManager()
.getAttributesForNode("host4").isEmpty());
// remove non existing attributes.
request = NodesToAttributesMappingRequest
.newInstance(AttributeMappingOperationType.REMOVE, ImmutableList
.of(NodeToAttributes.newInstance("host4", ImmutableList
.of(NodeAttribute
.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "x",
NodeAttributeType.STRING, "dfasdf")))), true);
try {
rm.adminService.mapAttributesToNodes(request);
fail("Should have failed for non exists attribute");
} catch (Exception ex) {
assertTrue("Exception expected if attributes does not exist", true);
}
request =
NodesToAttributesMappingRequest
.newInstance(AttributeMappingOperationType.ADD,