From 7435604a91a49f5c5717083fbaee74dd8ec1c426 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 20 Oct 2020 15:53:44 +0530 Subject: [PATCH] YARN-10453. Add partition resource info to get-node-labels and label-mappings api responses. Contributed Akhil PB. --- .../resourcemanager/webapp/NodeIDsInfo.java | 16 +++++ .../resourcemanager/webapp/RMWebServices.java | 18 ++++- .../webapp/dao/NodeLabelInfo.java | 10 +++ .../webapp/dao/PartitionInfo.java | 46 +++++++++++++ .../webapp/TestRMWebServicesNodeLabels.java | 67 ++++++++++++++++++- 5 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java index 5f45b9622d..b492793927 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java @@ -18,6 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; + import java.util.ArrayList; import java.util.List; @@ -39,6 +43,9 @@ public class NodeIDsInfo { @XmlElement(name="nodes") protected ArrayList nodeIDsList = new ArrayList(); + @XmlElement(name = "partitionInfo") + private PartitionInfo partitionInfo; + public NodeIDsInfo() { } // JAXB needs this @@ -46,7 +53,16 @@ public NodeIDsInfo(List nodeIdsList) { this.nodeIDsList.addAll(nodeIdsList); } + public NodeIDsInfo(List nodeIdsList, Resource resource) { + this(nodeIdsList); + this.partitionInfo = new PartitionInfo(new ResourceInfo(resource)); + } + public ArrayList getNodeIDs() { return nodeIDsList; } + + public PartitionInfo getPartitionInfo() { + return partitionInfo; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 02ad3ab5cd..2a725cf20c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -184,6 +184,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.RMQueueAclInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo; @@ -209,6 +210,7 @@ import org.apache.hadoop.yarn.util.AdHocLogDumper; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -1296,8 +1298,10 @@ public LabelsToNodesInfo getLabelsToNodes( for (NodeId nodeId : entry.getValue()) { nodeIdStrList.add(nodeId.toString()); } + Resource resource = rm.getRMContext().getNodeLabelManager() + .getResourceByLabel(entry.getKey().getName(), Resources.none()); ltsMap.put(new NodeLabelInfo(entry.getKey()), - new NodeIDsInfo(nodeIdStrList)); + new NodeIDsInfo(nodeIdStrList, resource)); } return lts; } @@ -1386,9 +1390,17 @@ public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr) List nodeLabels = rm.getRMContext().getNodeLabelManager().getClusterNodeLabels(); - NodeLabelsInfo ret = new NodeLabelsInfo(nodeLabels); - return ret; + ArrayList nodeLabelsInfo = new ArrayList(); + for (NodeLabel label: nodeLabels) { + Resource resource = rm.getRMContext().getNodeLabelManager() + .getResourceByLabel(label.getName(), Resources.none()); + PartitionInfo partitionInfo = + new PartitionInfo(new ResourceInfo(resource)); + nodeLabelsInfo.add(new NodeLabelInfo(label, partitionInfo)); + } + + return new NodeLabelsInfo(nodeLabelsInfo); } @POST diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java index e507c465e3..ce47471219 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java @@ -30,6 +30,7 @@ public class NodeLabelInfo { private String name; private boolean exclusivity; + private PartitionInfo partitionInfo; public NodeLabelInfo() { // JAXB needs this @@ -50,6 +51,11 @@ public NodeLabelInfo(NodeLabel label) { this.exclusivity = label.isExclusive(); } + public NodeLabelInfo(NodeLabel label, PartitionInfo partitionInfo) { + this(label); + this.partitionInfo = partitionInfo; + } + public String getName() { return name; } @@ -58,6 +64,10 @@ public boolean getExclusivity() { return exclusivity; } + public PartitionInfo getPartitionInfo() { + return partitionInfo; + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java new file mode 100644 index 0000000000..95baf74a18 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * XML element uses to represent partitionInfo. + */ +@XmlRootElement(name = "partitionInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class PartitionInfo { + + @XmlElement(name = "resourceAvailable") + private ResourceInfo resourceAvailable; + + public PartitionInfo() { + } + + public PartitionInfo(ResourceInfo resourceAvailable) { + this.resourceAvailable = resourceAvailable; + } + + public ResourceInfo getResourceAvailable() { + return resourceAvailable; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java index 0b2c6fe511..f1ea0fba03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -130,7 +131,8 @@ public void testNodeLabels() throws JSONException, Exception { r.path("ws").path("v1").path("cluster") .path("add-node-labels").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(nlsifo, NodeLabelsInfo.class), MediaType.APPLICATION_JSON) + .entity(toJson(nlsifo, NodeLabelsInfo.class), + MediaType.APPLICATION_JSON) .post(ClientResponse.class); // Verify @@ -154,7 +156,8 @@ public void testNodeLabels() throws JSONException, Exception { r.path("ws").path("v1").path("cluster") .path("add-node-labels").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(nlsifo, NodeLabelsInfo.class), MediaType.APPLICATION_JSON) + .entity(toJson(nlsifo, NodeLabelsInfo.class), + MediaType.APPLICATION_JSON) .post(ClientResponse.class); // Verify @@ -690,6 +693,66 @@ public void testLabelInvalidRemove() validateJsonExceptionContent(response, expectedmessage); } + @Test + public void testNodeLabelPartitionInfo() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Add a node label + NodeLabelsInfo nlsifo = new NodeLabelsInfo(); + nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("a")); + response = + r.path("ws").path("v1").path("cluster") + .path("add-node-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(nlsifo, NodeLabelsInfo.class), MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + + // Verify partition info in get-node-labels + response = + r.path("ws").path("v1").path("cluster") + .path("get-node-labels").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + nlsifo = response.getEntity(NodeLabelsInfo.class); + assertEquals(1, nlsifo.getNodeLabels().size()); + for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) { + assertEquals("a", nl.getName()); + assertTrue(nl.getExclusivity()); + assertNotNull(nl.getPartitionInfo()); + assertNotNull(nl.getPartitionInfo().getResourceAvailable()); + } + + // Add node label to a node + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("labels", "a"); + response = + r.path("ws").path("v1").path("cluster") + .path("nodes").path("nodeId:0") + .path("replace-labels") + .queryParam("user.name", userName) + .queryParams(params) + .accept(MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + + // Verify partition info in label-mappings + response = + r.path("ws").path("v1").path("cluster") + .path("label-mappings").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class); + assertEquals(1, ltni.getLabelsToNodes().size()); + NodeIDsInfo nodes = ltni.getLabelsToNodes().get( + new NodeLabelInfo("a")); + assertTrue(nodes.getNodeIDs().contains("nodeId:0")); + assertNotNull(nodes.getPartitionInfo()); + assertNotNull(nodes.getPartitionInfo().getResourceAvailable()); + } + @SuppressWarnings("rawtypes") private String toJson(Object nsli, Class klass) throws Exception { StringWriter sw = new StringWriter();