From 1a6ca75f3872587fb34c995a4b372b8cd6366d7d Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Wed, 22 Feb 2017 11:17:09 -0800 Subject: [PATCH] YARN-6143. Fix incompatible issue caused by YARN-3583. (Sunil G via wangda) --- .../hadoop/mapred/ResourceMgrDelegate.java | 6 +-- .../GetLabelsToNodesResponse.java | 6 +-- .../GetNodesToLabelsResponse.java | 7 ++- .../AddToClusterNodeLabelsRequest.java | 11 ++-- ...erver_resourcemanager_service_protos.proto | 5 +- .../src/main/proto/yarn_protos.proto | 6 +-- .../src/main/proto/yarn_service_protos.proto | 5 +- .../hadoop/yarn/client/api/YarnClient.java | 6 +-- .../yarn/client/api/impl/YarnClientImpl.java | 6 +-- .../yarn/client/api/impl/TestYarnClient.java | 49 ++++++++---------- .../GetClusterNodeLabelsResponsePBImpl.java | 4 ++ .../pb/GetLabelsToNodesResponsePBImpl.java | 20 +++----- .../pb/GetNodesToLabelsResponsePBImpl.java | 50 ++++++++----------- .../AddToClusterNodeLabelsRequestPBImpl.java | 47 +++++++++-------- .../pb/ReplaceLabelsOnNodeRequestPBImpl.java | 18 +++---- .../server/resourcemanager/AdminService.java | 7 +-- .../resourcemanager/ClientRMService.java | 6 +-- .../resourcemanager/TestClientRMService.java | 42 +++++----------- 18 files changed, 135 insertions(+), 166 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index c302553f36..62aa497292 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -476,19 +476,19 @@ public ReservationListResponse listReservations( return client.listReservations(request); } @Override - public Map> getNodeToLabels() throws YarnException, + public Map> getNodeToLabels() throws YarnException, IOException { return client.getNodeToLabels(); } @Override - public Map> getLabelsToNodes() throws YarnException, + public Map> getLabelsToNodes() throws YarnException, IOException { return client.getLabelsToNodes(); } @Override - public Map> getLabelsToNodes(Set labels) + public Map> getLabelsToNodes(Set labels) throws YarnException, IOException { return client.getLabelsToNodes(labels); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java index da2be28830..ef0bf60fa9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java @@ -29,7 +29,7 @@ public abstract class GetLabelsToNodesResponse { public static GetLabelsToNodesResponse newInstance( - Map> map) { + Map> map) { GetLabelsToNodesResponse response = Records.newRecord(GetLabelsToNodesResponse.class); response.setLabelsToNodes(map); @@ -38,9 +38,9 @@ public static GetLabelsToNodesResponse newInstance( @Public @Evolving - public abstract void setLabelsToNodes(Map> map); + public abstract void setLabelsToNodes(Map> map); @Public @Evolving - public abstract Map> getLabelsToNodes(); + public abstract Map> getLabelsToNodes(); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java index 432485c358..bcd5421e7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java @@ -24,12 +24,11 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.util.Records; public abstract class GetNodesToLabelsResponse { public static GetNodesToLabelsResponse newInstance( - Map> map) { + Map> map) { GetNodesToLabelsResponse response = Records.newRecord(GetNodesToLabelsResponse.class); response.setNodeToLabels(map); @@ -38,9 +37,9 @@ public static GetNodesToLabelsResponse newInstance( @Public @Evolving - public abstract void setNodeToLabels(Map> map); + public abstract void setNodeToLabels(Map> map); @Public @Evolving - public abstract Map> getNodeToLabels(); + public abstract Map> getNodeToLabels(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java index c1ea07d081..f2ac395dc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java @@ -30,17 +30,16 @@ public abstract class AddToClusterNodeLabelsRequest { @Public @Unstable - public static AddToClusterNodeLabelsRequest newInstance( - List NodeLabels) { - AddToClusterNodeLabelsRequest request = - Records.newRecord(AddToClusterNodeLabelsRequest.class); - request.setNodeLabels(NodeLabels); + public static AddToClusterNodeLabelsRequest newInstance(List nodeLabels) { + AddToClusterNodeLabelsRequest request = Records + .newRecord(AddToClusterNodeLabelsRequest.class); + request.setNodeLabels(nodeLabels); return request; } @Public @Unstable - public abstract void setNodeLabels(List NodeLabels); + public abstract void setNodeLabels(List nodeLabels); @Public @Unstable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index 16d80974c8..e8c92d962f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -84,7 +84,8 @@ message RefreshNodesResourcesResponseProto { } message AddToClusterNodeLabelsRequestProto { - repeated NodeLabelProto nodeLabels = 1; + repeated string deprecatedNodeLabels = 1; + repeated NodeLabelProto nodeLabels = 2; } message AddToClusterNodeLabelsResponseProto { @@ -98,7 +99,7 @@ message RemoveFromClusterNodeLabelsResponseProto { } message ReplaceLabelsOnNodeRequestProto { - repeated NodeIdToLabelsNameProto nodeToLabels = 1; + repeated NodeIdToLabelsProto nodeToLabels = 1; optional bool failOnUnknownNodes = 2; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index a6dbf3c290..3b26a5cb3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -282,13 +282,13 @@ message NodeReportProto { optional ResourceUtilizationProto node_utilization = 12; } -message NodeIdToLabelsInfoProto { +message NodeIdToLabelsProto { optional NodeIdProto nodeId = 1; - repeated NodeLabelProto nodeLabels = 2; + repeated string nodeLabels = 2; } message LabelsToNodeIdsProto { - optional NodeLabelProto nodeLabels = 1; + optional string nodeLabels = 1; repeated NodeIdProto nodeId = 2; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index c6647c825e..0e148966c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -234,7 +234,7 @@ message GetNodesToLabelsRequestProto { } message GetNodesToLabelsResponseProto { - repeated NodeIdToLabelsInfoProto nodeToLabels = 1; + repeated NodeIdToLabelsProto nodeToLabels = 1; } message GetLabelsToNodesRequestProto { @@ -249,7 +249,8 @@ message GetClusterNodeLabelsRequestProto { } message GetClusterNodeLabelsResponseProto { - repeated NodeLabelProto nodeLabels = 1; + repeated string deprecatedNodeLabels = 1; + repeated NodeLabelProto nodeLabels = 2; } message UpdateApplicationPriorityRequestProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 0bf16431de..8c68a31ffe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -767,7 +767,7 @@ public abstract ReservationListResponse listReservations( */ @Public @Unstable - public abstract Map> getNodeToLabels() + public abstract Map> getNodeToLabels() throws YarnException, IOException; /** @@ -782,7 +782,7 @@ public abstract Map> getNodeToLabels() */ @Public @Unstable - public abstract Map> getLabelsToNodes() + public abstract Map> getLabelsToNodes() throws YarnException, IOException; /** @@ -798,7 +798,7 @@ public abstract Map> getLabelsToNodes() */ @Public @Unstable - public abstract Map> getLabelsToNodes( + public abstract Map> getLabelsToNodes( Set labels) throws YarnException, IOException; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 23b128c414..8865b522d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -883,21 +883,21 @@ public ReservationListResponse listReservations( } @Override - public Map> getNodeToLabels() throws YarnException, + public Map> getNodeToLabels() throws YarnException, IOException { return rmClient.getNodeToLabels(GetNodesToLabelsRequest.newInstance()) .getNodeToLabels(); } @Override - public Map> getLabelsToNodes() throws YarnException, + public Map> getLabelsToNodes() throws YarnException, IOException { return rmClient.getLabelsToNodes(GetLabelsToNodesRequest.newInstance()) .getLabelsToNodes(); } @Override - public Map> getLabelsToNodes(Set labels) + public Map> getLabelsToNodes(Set labels) throws YarnException, IOException { return rmClient.getLabelsToNodes( GetLabelsToNodesRequest.newInstance(labels)).getLabelsToNodes(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index c2c966556d..9adc1872c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -609,9 +609,9 @@ public void testGetLabelsToNodes() throws YarnException, IOException { client.start(); // Get labels to nodes mapping - Map> expectedLabelsToNodes = + Map> expectedLabelsToNodes = ((MockYarnClient)client).getLabelsToNodesMap(); - Map> labelsToNodes = client.getLabelsToNodes(); + Map> labelsToNodes = client.getLabelsToNodes(); Assert.assertEquals(labelsToNodes, expectedLabelsToNodes); Assert.assertEquals(labelsToNodes.size(), 3); @@ -635,18 +635,12 @@ public void testGetNodesToLabels() throws YarnException, IOException { client.start(); // Get labels to nodes mapping - Map> expectedNodesToLabels = ((MockYarnClient) client) + Map> expectedNodesToLabels = ((MockYarnClient) client) .getNodeToLabelsMap(); - Map> nodesToLabels = client.getNodeToLabels(); + Map> nodesToLabels = client.getNodeToLabels(); Assert.assertEquals(nodesToLabels, expectedNodesToLabels); Assert.assertEquals(nodesToLabels.size(), 1); - // Verify exclusivity - Set labels = nodesToLabels.get(NodeId.newInstance("host", 0)); - for (NodeLabel label : labels) { - Assert.assertFalse(label.isExclusive()); - } - client.stop(); client.close(); } @@ -884,7 +878,7 @@ private List getApplicationReports( } @Override - public Map> getLabelsToNodes() + public Map> getLabelsToNodes() throws YarnException, IOException { when(mockLabelsToNodesResponse.getLabelsToNodes()).thenReturn( getLabelsToNodesMap()); @@ -892,48 +886,45 @@ public Map> getLabelsToNodes() } @Override - public Map> getLabelsToNodes(Set labels) + public Map> getLabelsToNodes(Set labels) throws YarnException, IOException { when(mockLabelsToNodesResponse.getLabelsToNodes()).thenReturn( getLabelsToNodesMap(labels)); return super.getLabelsToNodes(labels); } - public Map> getLabelsToNodesMap() { - Map> map = new HashMap>(); + public Map> getLabelsToNodesMap() { + Map> map = new HashMap>(); Set setNodeIds = new HashSet(Arrays.asList( NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0))); - map.put(NodeLabel.newInstance("x"), setNodeIds); - map.put(NodeLabel.newInstance("y"), setNodeIds); - map.put(NodeLabel.newInstance("z"), setNodeIds); + map.put("x", setNodeIds); + map.put("y", setNodeIds); + map.put("z", setNodeIds); return map; } - public Map> getLabelsToNodesMap(Set labels) { - Map> map = new HashMap>(); - Set setNodeIds = - new HashSet(Arrays.asList( + public Map> getLabelsToNodesMap(Set labels) { + Map> map = new HashMap>(); + Set setNodeIds = new HashSet(Arrays.asList( NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0))); - for(String label : labels) { - map.put(NodeLabel.newInstance(label), setNodeIds); + for (String label : labels) { + map.put(label, setNodeIds); } return map; } @Override - public Map> getNodeToLabels() throws YarnException, + public Map> getNodeToLabels() throws YarnException, IOException { when(mockNodeToLabelsResponse.getNodeToLabels()).thenReturn( getNodeToLabelsMap()); return super.getNodeToLabels(); } - public Map> getNodeToLabelsMap() { - Map> map = new HashMap>(); - Set setNodeLabels = new HashSet(Arrays.asList( - NodeLabel.newInstance("x", false), - NodeLabel.newInstance("y", false))); + public Map> getNodeToLabelsMap() { + Map> map = new HashMap>(); + Set setNodeLabels = new HashSet(Arrays.asList("x", "y")); map.put(NodeId.newInstance("host", 0), setNodeLabels); return map; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java index 227abe963b..54d454d398 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java @@ -72,11 +72,15 @@ private void mergeLocalToBuilder() { private void addNodeLabelsToProto() { maybeInitBuilder(); builder.clearNodeLabels(); + builder.clearDeprecatedNodeLabels(); List protoList = new ArrayList(); + List protoListString = new ArrayList(); for (NodeLabel r : this.updatedNodeLabels) { protoList.add(convertToProtoFormat(r)); + protoListString.add(r.getName()); } builder.addAllNodeLabels(protoList); + builder.addAllDeprecatedNodeLabels(protoListString); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java index 418fcbd4cc..961b9f1eec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java @@ -46,7 +46,7 @@ public class GetLabelsToNodesResponsePBImpl extends GetLabelsToNodesResponseProto.Builder builder = null; boolean viaProto = false; - private Map> labelsToNodes; + private Map> labelsToNodes; public GetLabelsToNodesResponsePBImpl() { this.builder = GetLabelsToNodesResponseProto.newBuilder(); @@ -63,7 +63,7 @@ private void initLabelsToNodes() { } GetLabelsToNodesResponseProtoOrBuilder p = viaProto ? proto : builder; List list = p.getLabelsToNodesList(); - this.labelsToNodes = new HashMap>(); + this.labelsToNodes = new HashMap>(); for (LabelsToNodeIdsProto c : list) { Set setNodes = new HashSet(); @@ -73,7 +73,7 @@ private void initLabelsToNodes() { } if (!setNodes.isEmpty()) { this.labelsToNodes - .put(new NodeLabelPBImpl(c.getNodeLabels()), setNodes); + .put(c.getNodeLabels(), setNodes); } } } @@ -97,7 +97,7 @@ private void addLabelsToNodesToProto() { public Iterator iterator() { return new Iterator() { - Iterator>> iter = + Iterator>> iter = labelsToNodes.entrySet().iterator(); @Override @@ -107,13 +107,13 @@ public void remove() { @Override public LabelsToNodeIdsProto next() { - Entry> now = iter.next(); + Entry> now = iter.next(); Set nodeProtoSet = new HashSet(); for(NodeId n : now.getValue()) { nodeProtoSet.add(convertToProtoFormat(n)); } return LabelsToNodeIdsProto.newBuilder() - .setNodeLabels(convertToProtoFormat(now.getKey())) + .setNodeLabels(now.getKey()) .addAllNodeId(nodeProtoSet) .build(); } @@ -153,10 +153,6 @@ private NodeIdProto convertToProtoFormat(NodeId t) { return ((NodeIdPBImpl)t).getProto(); } - private NodeLabelProto convertToProtoFormat(NodeLabel l) { - return ((NodeLabelPBImpl)l).getProto(); - } - @Override public int hashCode() { assert false : "hashCode not designed"; @@ -176,7 +172,7 @@ public boolean equals(Object other) { @Override @Public @Evolving - public void setLabelsToNodes(Map> map) { + public void setLabelsToNodes(Map> map) { initLabelsToNodes(); labelsToNodes.clear(); labelsToNodes.putAll(map); @@ -185,7 +181,7 @@ public void setLabelsToNodes(Map> map) { @Override @Public @Evolving - public Map> getLabelsToNodes() { + public Map> getLabelsToNodes() { initLabelsToNodes(); return this.labelsToNodes; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java index 52be73f6a4..26a6389454 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java @@ -32,11 +32,13 @@ import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsInfoProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProtoOrBuilder; +import com.google.common.collect.Sets; + public class GetNodesToLabelsResponsePBImpl extends GetNodesToLabelsResponse { GetNodesToLabelsResponseProto proto = GetNodesToLabelsResponseProto @@ -44,7 +46,7 @@ public class GetNodesToLabelsResponsePBImpl extends GetNodesToLabelsResponseProto.Builder builder = null; boolean viaProto = false; - private Map> nodeToLabels; + private Map> nodeToLabels; public GetNodesToLabelsResponsePBImpl() { this.builder = GetNodesToLabelsResponseProto.newBuilder(); @@ -60,15 +62,12 @@ private void initNodeToLabels() { return; } GetNodesToLabelsResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getNodeToLabelsList(); - this.nodeToLabels = new HashMap>(); + List list = p.getNodeToLabelsList(); + this.nodeToLabels = new HashMap>(); - for (NodeIdToLabelsInfoProto c : list) { - Set labels = new HashSet(); - for (NodeLabelProto l : c.getNodeLabelsList()) { - labels.add(new NodeLabelPBImpl(l)); - } - this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()), labels); + for (NodeIdToLabelsProto c : list) { + this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()), + Sets.newHashSet(c.getNodeLabelsList())); } } @@ -85,13 +84,13 @@ private void addNodeToLabelsToProto() { if (nodeToLabels == null) { return; } - Iterable iterable = - new Iterable() { + Iterable iterable = + new Iterable() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { - Iterator>> iter = nodeToLabels + Iterator>> iter = nodeToLabels .entrySet().iterator(); @Override @@ -100,16 +99,11 @@ public void remove() { } @Override - public NodeIdToLabelsInfoProto next() { - Entry> now = iter.next(); - Set labelProtoList = - new HashSet(); - for (NodeLabel l : now.getValue()) { - labelProtoList.add(convertToProtoFormat(l)); - } - return NodeIdToLabelsInfoProto.newBuilder() + public NodeIdToLabelsProto next() { + Entry> now = iter.next(); + return NodeIdToLabelsProto.newBuilder() .setNodeId(convertToProtoFormat(now.getKey())) - .addAllNodeLabels(labelProtoList).build(); + .addAllNodeLabels(now.getValue()).build(); } @Override @@ -144,13 +138,13 @@ public GetNodesToLabelsResponseProto getProto() { } @Override - public Map> getNodeToLabels() { + public Map> getNodeToLabels() { initNodeToLabels(); return this.nodeToLabels; } @Override - public void setNodeToLabels(Map> map) { + public void setNodeToLabels(Map> map) { initNodeToLabels(); nodeToLabels.clear(); nodeToLabels.putAll(map); @@ -159,10 +153,6 @@ public void setNodeToLabels(Map> map) { private NodeIdProto convertToProtoFormat(NodeId t) { return ((NodeIdPBImpl)t).getProto(); } - - private NodeLabelProto convertToProtoFormat(NodeLabel t) { - return ((NodeLabelPBImpl)t).getProto(); - } @Override public int hashCode() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java index 1ff0beff66..0f1e400a98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import org.apache.hadoop.yarn.api.records.NodeLabel; @@ -30,7 +29,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; public class AddToClusterNodeLabelsRequestPBImpl extends - AddToClusterNodeLabelsRequest { + AddToClusterNodeLabelsRequest { AddToClusterNodeLabelsRequestProto proto = AddToClusterNodeLabelsRequestProto .getDefaultInstance(); AddToClusterNodeLabelsRequestProto.Builder builder = null; @@ -71,11 +70,15 @@ private void mergeLocalToBuilder() { private void addNodeLabelsToProto() { maybeInitBuilder(); builder.clearNodeLabels(); + builder.clearDeprecatedNodeLabels(); List protoList = new ArrayList(); + List protoListString = new ArrayList(); for (NodeLabel r : this.updatedNodeLabels) { protoList.add(convertToProtoFormat(r)); + protoListString.add(r.getName()); } builder.addAllNodeLabels(protoList); + builder.addAllDeprecatedNodeLabels(protoListString); } @Override @@ -101,17 +104,6 @@ private void maybeInitBuilder() { viaProto = false; } - @Override - public void setNodeLabels(List updatedNodeLabels) { - maybeInitBuilder(); - this.updatedNodeLabels = new ArrayList<>(); - if (updatedNodeLabels == null) { - builder.clearNodeLabels(); - return; - } - this.updatedNodeLabels.addAll(updatedNodeLabels); - } - private void initLocalNodeLabels() { AddToClusterNodeLabelsRequestProtoOrBuilder p = viaProto ? proto : builder; List attributesProtoList = p.getNodeLabelsList(); @@ -121,15 +113,6 @@ private void initLocalNodeLabels() { } } - @Override - public List getNodeLabels() { - if (this.updatedNodeLabels != null) { - return this.updatedNodeLabels; - } - initLocalNodeLabels(); - return this.updatedNodeLabels; - } - private NodeLabel convertFromProtoFormat(NodeLabelProto p) { return new NodeLabelPBImpl(p); } @@ -142,4 +125,24 @@ private NodeLabelProto convertToProtoFormat(NodeLabel t) { public String toString() { return getProto().toString(); } + + @Override + public void setNodeLabels(List nodeLabels) { + maybeInitBuilder(); + this.updatedNodeLabels = new ArrayList<>(); + if (nodeLabels == null) { + builder.clearNodeLabels(); + return; + } + this.updatedNodeLabels.addAll(nodeLabels); + } + + @Override + public List getNodeLabels() { + if (this.updatedNodeLabels != null) { + return this.updatedNodeLabels; + } + initLocalNodeLabels(); + return this.updatedNodeLabels; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java index 3b15b27d78..f6634b4f68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeIdToLabelsNameProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; @@ -58,10 +58,10 @@ private void initNodeToLabels() { return; } ReplaceLabelsOnNodeRequestProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getNodeToLabelsList(); + List list = p.getNodeToLabelsList(); this.nodeIdToLabels = new HashMap>(); - for (NodeIdToLabelsNameProto c : list) { + for (NodeIdToLabelsProto c : list) { this.nodeIdToLabels.put(new NodeIdPBImpl(c.getNodeId()), Sets.newHashSet(c.getNodeLabelsList())); } @@ -80,11 +80,11 @@ private void addNodeToLabelsToProto() { if (nodeIdToLabels == null) { return; } - Iterable iterable = - new Iterable() { + Iterable iterable = + new Iterable() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { Iterator>> iter = nodeIdToLabels .entrySet().iterator(); @@ -95,9 +95,9 @@ public void remove() { } @Override - public NodeIdToLabelsNameProto next() { + public NodeIdToLabelsProto next() { Entry> now = iter.next(); - return NodeIdToLabelsNameProto.newBuilder() + return NodeIdToLabelsProto.newBuilder() .setNodeId(convertToProtoFormat(now.getKey())).clearNodeLabels() .addAllNodeLabels(now.getValue()).build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 86bb896d88..9dda57e2d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -735,9 +735,10 @@ public AddToClusterNodeLabelsResponse addToClusterNodeLabels(AddToClusterNodeLab AddToClusterNodeLabelsResponse response = recordFactory.newRecordInstance(AddToClusterNodeLabelsResponse.class); try { - rmContext.getNodeLabelManager().addToCluserNodeLabels(request.getNodeLabels()); - RMAuditLogger - .logSuccess(user.getShortUserName(), operation, "AdminService"); + rmContext.getNodeLabelManager() + .addToCluserNodeLabels(request.getNodeLabels()); + RMAuditLogger.logSuccess(user.getShortUserName(), operation, + "AdminService"); return response; } catch (IOException ioe) { throw logAndWrapException(ioe, user.getShortUserName(), operation, msg); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index c4e3bf7453..0c87ede777 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1490,7 +1490,7 @@ public GetNodesToLabelsResponse getNodeToLabels( GetNodesToLabelsRequest request) throws YarnException, IOException { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); GetNodesToLabelsResponse response = - GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabelsInfo()); + GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabels()); return response; } @@ -1500,10 +1500,10 @@ public GetLabelsToNodesResponse getLabelsToNodes( RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); if (request.getNodeLabels() == null || request.getNodeLabels().isEmpty()) { return GetLabelsToNodesResponse.newInstance( - labelsMgr.getLabelsInfoToNodes()); + labelsMgr.getLabelsToNodes()); } else { return GetLabelsToNodesResponse.newInstance( - labelsMgr.getLabelsInfoToNodes(request.getNodeLabels())); + labelsMgr.getLabelsToNodes(request.getNodeLabels())); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 637629059d..d5f9accd34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -1799,27 +1799,19 @@ protected ClientRMService createClientRMService() { // Get node labels mapping GetNodesToLabelsResponse response1 = client .getNodeToLabels(GetNodesToLabelsRequest.newInstance()); - Map> nodeToLabels = response1.getNodeToLabels(); + Map> nodeToLabels = response1.getNodeToLabels(); Assert.assertTrue(nodeToLabels.keySet().containsAll( Arrays.asList(node1, node2))); Assert.assertTrue(nodeToLabels.get(node1) - .containsAll(Arrays.asList(labelX))); + .containsAll(Arrays.asList(labelX.getName()))); Assert.assertTrue(nodeToLabels.get(node2) - .containsAll(Arrays.asList(labelY))); - // Verify whether labelX's exclusivity is false - for (NodeLabel x : nodeToLabels.get(node1)) { - Assert.assertFalse(x.isExclusive()); - } - // Verify whether labelY's exclusivity is true - for (NodeLabel y : nodeToLabels.get(node2)) { - Assert.assertTrue(y.isExclusive()); - } + .containsAll(Arrays.asList(labelY.getName()))); // Below label "x" is not present in the response as exclusivity is true Assert.assertFalse(nodeToLabels.get(node1).containsAll( Arrays.asList(NodeLabel.newInstance("x")))); rpc.stopProxy(client, conf); - rm.close(); + rm.stop(); } @Test @@ -1870,18 +1862,14 @@ protected ClientRMService createClientRMService() { // Get labels to nodes mapping GetLabelsToNodesResponse response1 = client .getLabelsToNodes(GetLabelsToNodesRequest.newInstance()); - Map> labelsToNodes = response1.getLabelsToNodes(); - // Verify whether all NodeLabel's exclusivity are false - for (Map.Entry> nltn : labelsToNodes.entrySet()) { - Assert.assertFalse(nltn.getKey().isExclusive()); - } + Map> labelsToNodes = response1.getLabelsToNodes(); Assert.assertTrue(labelsToNodes.keySet().containsAll( - Arrays.asList(labelX, labelY, labelZ))); - Assert.assertTrue(labelsToNodes.get(labelX).containsAll( + Arrays.asList(labelX.getName(), labelY.getName(), labelZ.getName()))); + Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll( Arrays.asList(node1A))); - Assert.assertTrue(labelsToNodes.get(labelY).containsAll( + Assert.assertTrue(labelsToNodes.get(labelY.getName()).containsAll( Arrays.asList(node2A, node3A))); - Assert.assertTrue(labelsToNodes.get(labelZ).containsAll( + Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll( Arrays.asList(node1B, node3B))); // Get labels to nodes mapping for specific labels @@ -1890,17 +1878,13 @@ protected ClientRMService createClientRMService() { GetLabelsToNodesResponse response2 = client .getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels)); labelsToNodes = response2.getLabelsToNodes(); - // Verify whether all NodeLabel's exclusivity are false - for (Map.Entry> nltn : labelsToNodes.entrySet()) { - Assert.assertFalse(nltn.getKey().isExclusive()); - } Assert.assertTrue(labelsToNodes.keySet().containsAll( - Arrays.asList(labelX, labelZ))); - Assert.assertTrue(labelsToNodes.get(labelX).containsAll( + Arrays.asList(labelX.getName(), labelZ.getName()))); + Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll( Arrays.asList(node1A))); - Assert.assertTrue(labelsToNodes.get(labelZ).containsAll( + Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll( Arrays.asList(node1B, node3B))); - Assert.assertEquals(labelsToNodes.get(labelY), null); + Assert.assertEquals(labelsToNodes.get(labelY.getName()), null); rpc.stopProxy(client, conf); rm.close();