From 8cf6a9a2bde9d4ccad95a514885634792889ec7f Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Sun, 13 May 2018 09:35:11 +0530 Subject: [PATCH] YARN-7892. Revisit NodeAttribute class structure. Contributed by Naganarasimha G R. --- .../hadoop/mapred/ResourceMgrDelegate.java | 9 +- .../GetAttributesToNodesRequest.java | 36 +++-- .../GetAttributesToNodesResponse.java | 35 +++-- .../GetClusterNodeAttributesResponse.java | 25 +-- .../yarn/api/records/NodeAttribute.java | 17 +-- .../yarn/api/records/NodeAttributeInfo.java | 62 ++++++++ .../yarn/api/records/NodeAttributeKey.java | 66 ++++++++ .../api/records/NodeToAttributeValue.java | 57 +++++++ .../src/main/proto/yarn_protos.proto | 27 +++- .../src/main/proto/yarn_service_protos.proto | 4 +- .../hadoop/yarn/client/api/YarnClient.java | 22 +-- .../yarn/client/api/impl/YarnClientImpl.java | 9 +- .../pb/GetAttributesToNodesRequestPBImpl.java | 45 +++--- .../GetAttributesToNodesResponsePBImpl.java | 83 ++++++---- ...etClusterNodeAttributesResponsePBImpl.java | 68 +++++---- .../impl/pb/NodeAttributeInfoPBImpl.java | 143 ++++++++++++++++++ .../impl/pb/NodeAttributeKeyPBImpl.java | 140 +++++++++++++++++ .../records/impl/pb/NodeAttributePBImpl.java | 79 ++++------ .../impl/pb/NodeToAttributeValuePBImpl.java | 137 +++++++++++++++++ .../nodelabels/NodeAttributesManager.java | 16 +- .../hadoop/yarn/nodelabels/NodeLabelUtil.java | 14 +- .../yarn/nodelabels/RMNodeAttribute.java | 18 +-- .../hadoop/yarn/api/TestPBImplRecords.java | 45 +++++- ...stConfigurationNodeAttributesProvider.java | 17 ++- ...TestScriptBasedNodeAttributesProvider.java | 5 +- .../server/resourcemanager/AdminService.java | 2 +- .../resourcemanager/ClientRMService.java | 31 +++- .../ResourceTrackerService.java | 2 +- .../nodelabels/NodeAttributesManagerImpl.java | 114 ++++++++------ .../nodelabels/NodeLabelsUtils.java | 19 +++ .../webapp/dao/NodeAttributeInfo.java | 4 +- .../resourcemanager/TestClientRMService.java | 108 +++++++------ .../TestResourceTrackerService.java | 4 +- .../TestFileSystemNodeAttributeStore.java | 6 +- .../nodelabels/TestNodeAttributesManager.java | 9 +- .../clientrm/FederationClientInterceptor.java | 2 +- 36 files changed, 1121 insertions(+), 359 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeKey.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeToAttributeValue.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributeInfoPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributeKeyPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeToAttributeValuePBImpl.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 1a7f308f56..2cb37166c9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -59,10 +59,13 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; @@ -541,14 +544,14 @@ public List getResourceTypeInfo() } @Override - public Set getClusterAttributes() + public Set getClusterAttributes() throws YarnException, IOException { return client.getClusterAttributes(); } @Override - public Map> getAttributesToNodes( - Set attributes) throws YarnException, IOException { + public Map> getAttributesToNodes( + Set attributes) throws YarnException, IOException { return client.getAttributesToNodes(attributes); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java index d9531b0c38..94814e9053 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java @@ -17,22 +17,24 @@ */ package org.apache.hadoop.yarn.api.protocolrecords; -import static org.apache.hadoop.classification.InterfaceAudience.*; -import static org.apache.hadoop.classification.InterfaceStability.*; -import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.records.NodeAttribute; -import org.apache.hadoop.yarn.util.Records; - import java.util.Set; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.util.Records; + /** *

- * The request from clients to get attribtues to nodes mapping - * in the cluster from the ResourceManager. + * The request from clients to get node to attribute value mapping for all or + * given set of Node AttributeKey's in the cluster from the + * ResourceManager. *

* * @see ApplicationClientProtocol#getAttributesToNodes - * (GetAttributesToNodesRequest) + * (GetAttributesToNodesRequest) */ @Public @Evolving @@ -43,7 +45,7 @@ public static GetAttributesToNodesRequest newInstance() { } public static GetAttributesToNodesRequest newInstance( - Set attributes) { + Set attributes) { GetAttributesToNodesRequest request = Records.newRecord(GetAttributesToNodesRequest.class); request.setNodeAttributes(attributes); @@ -51,20 +53,22 @@ public static GetAttributesToNodesRequest newInstance( } /** - * Set node attributes for which the mapping is required. + * Set node attributeKeys for which the mapping of hostname to attribute value + * is required. * - * @param attributes Set provided. + * @param attributes Set provided. */ @Public @Unstable - public abstract void setNodeAttributes(Set attributes); + public abstract void setNodeAttributes(Set attributes); /** - * Get node attributes for which mapping mapping is required. + * Get node attributeKeys for which mapping of hostname to attribute value is + * required. * - * @return Set + * @return Set */ @Public @Unstable - public abstract Set getNodeAttributes(); + public abstract Set getNodeAttributes(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java index 4fdb1f70a9..9bd529f3c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java @@ -17,29 +17,30 @@ */ package org.apache.hadoop.yarn.api.protocolrecords; -import static org.apache.hadoop.classification.InterfaceAudience.Public; -import static org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.records.NodeAttribute; -import org.apache.hadoop.yarn.util.Records; - +import java.util.List; import java.util.Map; -import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; +import org.apache.hadoop.yarn.util.Records; /** *

* The response sent by the ResourceManager to a client requesting - * attributes to hostname mapping. + * node to attribute value mapping for all or given set of Node AttributeKey's. *

* * @see ApplicationClientProtocol#getAttributesToNodes - * (GetAttributesToNodesRequest) + * (GetAttributesToNodesRequest) */ @Public @Evolving public abstract class GetAttributesToNodesResponse { public static GetAttributesToNodesResponse newInstance( - Map> map) { + Map> map) { GetAttributesToNodesResponse response = Records.newRecord(GetAttributesToNodesResponse.class); response.setAttributeToNodes(map); @@ -48,15 +49,17 @@ public static GetAttributesToNodesResponse newInstance( @Public @Evolving - public abstract void setAttributeToNodes(Map> map); + public abstract void setAttributeToNodes( + Map> map); - /* - * Get attributes to node hostname mapping. + /** + * Get mapping of NodeAttributeKey to its associated mapping of list of + * NodeToAttributeValuenode to attribute value. * - * @return Map> node attributes to hostname - * mapping. + * @return Map> node attributes + * to list of NodeToAttributeValuenode. */ @Public @Evolving - public abstract Map> getAttributesToNodes(); + public abstract Map> getAttributesToNodes(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java index cc3cae4c89..b0ccd906a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java @@ -17,14 +17,15 @@ */ package org.apache.hadoop.yarn.api.protocolrecords; -import static org.apache.hadoop.classification.InterfaceAudience.*; -import static org.apache.hadoop.classification.InterfaceStability.*; -import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.records.NodeAttribute; -import org.apache.hadoop.yarn.util.Records; - import java.util.Set; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; +import org.apache.hadoop.yarn.util.Records; + /** *

* The response sent by the ResourceManager to a client requesting @@ -45,7 +46,7 @@ public abstract class GetClusterNodeAttributesResponse { * @return GetClusterNodeAttributesResponse. */ public static GetClusterNodeAttributesResponse newInstance( - Set attributes) { + Set attributes) { GetClusterNodeAttributesResponse response = Records.newRecord(GetClusterNodeAttributesResponse.class); response.setNodeAttributes(attributes); @@ -55,18 +56,18 @@ public static GetClusterNodeAttributesResponse newInstance( /** * Set node attributes to the response. * - * @param attributes Node attributes + * @param attributes Map of Node attributeKey to Type. */ @Public @Unstable - public abstract void setNodeAttributes(Set attributes); + public abstract void setNodeAttributes(Set attributes); /** - * Get node attributes of the response. + * Get node attributes from the response. * - * @return Node attributes + * @return Node attributes. */ @Public @Unstable - public abstract Set getNodeAttributes(); + public abstract Set getNodeAttributes(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java index 25ac9abd59..7064939082 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java @@ -58,8 +58,9 @@ public static NodeAttribute newInstance(String attributePrefix, String attributeName, NodeAttributeType attributeType, String attributeValue) { NodeAttribute nodeAttribute = Records.newRecord(NodeAttribute.class); - nodeAttribute.setAttributePrefix(attributePrefix); - nodeAttribute.setAttributeName(attributeName); + NodeAttributeKey nodeAttributeKey = + NodeAttributeKey.newInstance(attributePrefix, attributeName); + nodeAttribute.setAttributeKey(nodeAttributeKey); nodeAttribute.setAttributeType(attributeType); nodeAttribute.setAttributeValue(attributeValue); return nodeAttribute; @@ -67,19 +68,11 @@ public static NodeAttribute newInstance(String attributePrefix, @Public @Unstable - public abstract String getAttributePrefix(); + public abstract NodeAttributeKey getAttributeKey(); @Public @Unstable - public abstract void setAttributePrefix(String attributePrefix); - - @Public - @Unstable - public abstract String getAttributeName(); - - @Public - @Unstable - public abstract void setAttributeName(String attributeName); + public abstract void setAttributeKey(NodeAttributeKey attributeKey); @Public @Unstable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeInfo.java new file mode 100644 index 0000000000..d294333ed1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeInfo.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * Node Attribute Info describes a NodeAttribute. + *

+ */ +@Public +@Unstable +public abstract class NodeAttributeInfo { + + public static NodeAttributeInfo newInstance(NodeAttribute nodeAttribute) { + return newInstance(nodeAttribute.getAttributeKey(), + nodeAttribute.getAttributeType()); + } + + public static NodeAttributeInfo newInstance(NodeAttributeKey nodeAttributeKey, + NodeAttributeType attributeType) { + NodeAttributeInfo nodeAttribute = + Records.newRecord(NodeAttributeInfo.class); + nodeAttribute.setAttributeKey(nodeAttributeKey); + nodeAttribute.setAttributeType(attributeType); + return nodeAttribute; + } + + @Public + @Unstable + public abstract NodeAttributeKey getAttributeKey(); + + @Public + @Unstable + public abstract void setAttributeKey(NodeAttributeKey attributeKey); + + @Public + @Unstable + public abstract NodeAttributeType getAttributeType(); + + @Public + @Unstable + public abstract void setAttributeType(NodeAttributeType attributeType); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeKey.java new file mode 100644 index 0000000000..35ff26f07f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeKey.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * Node AttributeKey uniquely identifies a given Node Attribute. Node Attribute + * is identified based on attribute prefix and name. + *

+ *

+ * Node Attribute Prefix is used as namespace to segregate the attributes. + *

+ */ +@Public +@Unstable +public abstract class NodeAttributeKey { + + public static NodeAttributeKey newInstance(String attributeName) { + return newInstance(NodeAttribute.PREFIX_CENTRALIZED, attributeName); + } + + public static NodeAttributeKey newInstance(String attributePrefix, + String attributeName) { + NodeAttributeKey nodeAttributeKey = + Records.newRecord(NodeAttributeKey.class); + nodeAttributeKey.setAttributePrefix(attributePrefix); + nodeAttributeKey.setAttributeName(attributeName); + return nodeAttributeKey; + } + + @Public + @Unstable + public abstract String getAttributePrefix(); + + @Public + @Unstable + public abstract void setAttributePrefix(String attributePrefix); + + @Public + @Unstable + public abstract String getAttributeName(); + + @Public + @Unstable + public abstract void setAttributeName(String attributeName); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeToAttributeValue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeToAttributeValue.java new file mode 100644 index 0000000000..0bcb8b68b4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeToAttributeValue.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * Mapping of Attribute Value to a Node. + *

+ */ +@Public +@Unstable +public abstract class NodeToAttributeValue { + public static NodeToAttributeValue newInstance(String hostname, + String attributeValue) { + NodeToAttributeValue nodeToAttributeValue = + Records.newRecord(NodeToAttributeValue.class); + nodeToAttributeValue.setAttributeValue(attributeValue); + nodeToAttributeValue.setHostname(hostname); + return nodeToAttributeValue; + } + + @Public + @Unstable + public abstract String getAttributeValue(); + + @Public + @Unstable + public abstract void setAttributeValue(String attributeValue); + + @Public + @Unstable + public abstract String getHostname(); + + @Public + @Unstable + public abstract void setHostname(String hostname); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5576ee6fb9..aca9471f6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -376,16 +376,31 @@ enum NodeAttributeTypeProto { STRING = 1; } -message NodeAttributeProto { - optional string attributePrefix = 1; +message NodeAttributeKeyProto { + optional string attributePrefix = 1 [default="rm.yarn.io"]; required string attributeName = 2; - optional NodeAttributeTypeProto attributeType = 3 [default = STRING]; - optional string attributeValue = 4 [default=""]; +} + +message NodeAttributeProto { + required NodeAttributeKeyProto attributeKey = 1; + optional NodeAttributeTypeProto attributeType = 2 [default = STRING]; + optional string attributeValue = 3 [default=""]; +} + + +message NodeAttributeInfoProto { + required NodeAttributeKeyProto attributeKey = 1; + required NodeAttributeTypeProto attributeType = 2; +} + +message NodeToAttributeValueProto { + required string hostname = 1; + required string attributeValue = 2; } message AttributeToNodesProto { - required NodeAttributeProto nodeAttribute = 1; - repeated string hostnames = 2; + required NodeAttributeKeyProto nodeAttribute = 1; + repeated NodeToAttributeValueProto nodeValueMap = 2; } message NodeToAttributesProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 439780b82c..248f775bde 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -264,11 +264,11 @@ message GetClusterNodeAttributesRequestProto { } message GetClusterNodeAttributesResponseProto { - repeated NodeAttributeProto nodeAttributes = 1; + repeated NodeAttributeInfoProto nodeAttributes = 1; } message GetAttributesToNodesRequestProto { - repeated NodeAttributeProto nodeAttributes = 1; + repeated NodeAttributeKeyProto nodeAttributes = 1; } message GetAttributesToNodesResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 0099845a5c..f51b2f9b16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -52,10 +52,13 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; @@ -914,26 +917,27 @@ public abstract List getResourceTypeInfo() */ @Public @Unstable - public abstract Set getClusterAttributes() + public abstract Set getClusterAttributes() throws YarnException, IOException; /** *

- * The interface used by client to get Attributes to nodes mapping - * for specified node attributes in existing cluster. + * The interface used by client to get mapping of AttributeKey to associated + * NodeToAttributeValue list for specified node attributeKeys in the cluster. *

* - * @param attributes Attributes for which Attributes to nodes mapping has to - * be retrieved.If empty or null is set then will return - * all attributes to node mapping in cluster. - * @return Attributes to nodes mappings for specific Attributes. + * @param attributes AttributeKeys for which associated NodeToAttributeValue + * mapping value has to be retrieved. If empty or null is set then + * will return mapping for all attributeKeys in the cluster + * @return mapping of AttributeKey to List of associated + * NodeToAttributeValue's. * @throws YarnException * @throws IOException */ @Public @Unstable - public abstract Map> getAttributesToNodes( - Set attributes) throws YarnException, IOException; + public abstract Map> getAttributesToNodes( + Set attributes) throws YarnException, IOException; /** *

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index a08d35d820..acfc3ff70b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -99,10 +99,13 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; @@ -981,7 +984,7 @@ public List getResourceTypeInfo() } @Override - public Set getClusterAttributes() + public Set getClusterAttributes() throws YarnException, IOException { GetClusterNodeAttributesRequest request = GetClusterNodeAttributesRequest.newInstance(); @@ -989,8 +992,8 @@ public Set getClusterAttributes() } @Override - public Map> getAttributesToNodes( - Set attributes) throws YarnException, IOException { + public Map> getAttributesToNodes( + Set attributes) throws YarnException, IOException { GetAttributesToNodesRequest request = GetAttributesToNodesRequest.newInstance(attributes); return rmClient.getAttributesToNodes(request).getAttributesToNodes(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java index a84fb44fc9..15a360cf63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java @@ -17,21 +17,21 @@ */ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; -import com.google.protobuf.TextFormat; -import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; -import org.apache.hadoop.yarn.api.records.NodeAttribute; -import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesRequestProto; - import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; -import static org.apache.hadoop.classification.InterfaceAudience.*; -import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeKeyPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesRequestProto; + +import com.google.protobuf.TextFormat; /** * Attributes to nodes mapping request. @@ -41,7 +41,7 @@ public class GetAttributesToNodesRequestPBImpl extends GetAttributesToNodesRequest { - private Set nodeAttributes = null; + private Set nodeAttributes = null; private GetAttributesToNodesRequestProto proto = GetAttributesToNodesRequestProto.getDefaultInstance(); @@ -86,9 +86,9 @@ private void addLocalAttributesToProto() { if (nodeAttributes == null) { return; } - Iterable iterable = - () -> new Iterator() { - private Iterator iter = nodeAttributes.iterator(); + Iterable iterable = + () -> new Iterator() { + private Iterator iter = nodeAttributes.iterator(); @Override public boolean hasNext() { @@ -96,7 +96,7 @@ public boolean hasNext() { } @Override - public NodeAttributeProto next() { + public NodeAttributeKeyProto next() { return convertToProtoFormat(iter.next()); } @@ -110,12 +110,13 @@ public void remove() { builder.addAllNodeAttributes(iterable); } - private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) { - return new NodeAttributePBImpl(p); + private NodeAttributeKeyPBImpl convertFromProtoFormat( + NodeAttributeKeyProto p) { + return new NodeAttributeKeyPBImpl(p); } - private NodeAttributeProto convertToProtoFormat(NodeAttribute t) { - return ((NodeAttributePBImpl) t).getProto(); + private NodeAttributeKeyProto convertToProtoFormat(NodeAttributeKey t) { + return ((NodeAttributeKeyPBImpl) t).getProto(); } private void maybeInitBuilder() { @@ -131,7 +132,7 @@ private void initNodeAttributes() { } YarnServiceProtos.GetAttributesToNodesRequestProtoOrBuilder p = viaProto ? proto : builder; - List nodeAttributesList = p.getNodeAttributesList(); + List nodeAttributesList = p.getNodeAttributesList(); this.nodeAttributes = new HashSet<>(); nodeAttributesList .forEach((v) -> nodeAttributes.add(convertFromProtoFormat(v))); @@ -159,7 +160,7 @@ public String toString() { } @Override - public void setNodeAttributes(Set attributes) { + public void setNodeAttributes(Set attributes) { maybeInitBuilder(); if (nodeAttributes == null) { builder.clearNodeAttributes(); @@ -168,7 +169,7 @@ public void setNodeAttributes(Set attributes) { } @Override - public Set getNodeAttributes() { + public Set getNodeAttributes() { initNodeAttributes(); return this.nodeAttributes; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java index 175c10e524..11999bdc1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java @@ -17,23 +17,24 @@ */ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; -import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; -import org.apache.hadoop.yarn.api.records.NodeAttribute; -import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.AttributeToNodesProto; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto; - +import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; -import static org.apache.hadoop.classification.InterfaceAudience.*; -import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeKeyPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeToAttributeValuePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.AttributeToNodesProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributeValueProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto; /** * Attributes to nodes response. @@ -48,7 +49,7 @@ public class GetAttributesToNodesResponsePBImpl private GetAttributesToNodesResponseProto.Builder builder = null; private boolean viaProto = false; - private Map> attributesToNodes; + private Map> attributesToNodes; public GetAttributesToNodesResponsePBImpl() { this.builder = GetAttributesToNodesResponseProto.newBuilder(); @@ -70,10 +71,15 @@ private void initAttributesToNodes() { this.attributesToNodes = new HashMap<>(); for (AttributeToNodesProto c : list) { - Set setNodes = new HashSet<>(c.getHostnamesList()); - if (!setNodes.isEmpty()) { - this.attributesToNodes - .put(convertFromProtoFormat(c.getNodeAttribute()), setNodes); + List nodeValueMapList = + c.getNodeValueMapList(); + List nodeToAttributeValue = new ArrayList<>(); + for (NodeToAttributeValueProto valueProto : nodeValueMapList) { + nodeToAttributeValue.add(convertFromProtoFormat(valueProto)); + } + if (!nodeToAttributeValue.isEmpty()) { + this.attributesToNodes.put(convertFromProtoFormat(c.getNodeAttribute()), + nodeToAttributeValue); } } } @@ -94,7 +100,7 @@ private void addAttributesToNodesToProto() { Iterable iterable = () -> new Iterator() { - private Iterator>> iter = + private Iterator>> iter = attributesToNodes.entrySet().iterator(); @Override @@ -104,14 +110,18 @@ public void remove() { @Override public AttributeToNodesProto next() { - Map.Entry> now = iter.next(); - Set hostNames = new HashSet<>(); - for (String host : now.getValue()) { - hostNames.add(host); + Map.Entry> attrToNodes + = iter.next(); + + AttributeToNodesProto.Builder attrToNodesBuilder = + AttributeToNodesProto.newBuilder().setNodeAttribute( + convertToProtoFormat(attrToNodes.getKey())); + for (NodeToAttributeValue hostToAttrVal : attrToNodes.getValue()) { + attrToNodesBuilder + .addNodeValueMap(convertToProtoFormat(hostToAttrVal)); } - return AttributeToNodesProto.newBuilder() - .setNodeAttribute(convertToProtoFormat(now.getKey())) - .addAllHostnames(hostNames).build(); + + return attrToNodesBuilder.build(); } @Override @@ -122,12 +132,22 @@ public boolean hasNext() { builder.addAllAttributesToNodes(iterable); } - private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) { - return new NodeAttributePBImpl(p); + private NodeAttributeKey convertFromProtoFormat(NodeAttributeKeyProto p) { + return new NodeAttributeKeyPBImpl(p); } - private NodeAttributeProto convertToProtoFormat(NodeAttribute t) { - return ((NodeAttributePBImpl) t).getProto(); + private NodeAttributeKeyProto convertToProtoFormat(NodeAttributeKey t) { + return ((NodeAttributeKeyPBImpl) t).getProto(); + } + + private NodeToAttributeValue convertFromProtoFormat( + NodeToAttributeValueProto p) { + return new NodeToAttributeValuePBImpl(p); + } + + private NodeToAttributeValueProto convertToProtoFormat( + NodeToAttributeValue t) { + return ((NodeToAttributeValuePBImpl) t).getProto(); } private void mergeLocalToBuilder() { @@ -170,14 +190,15 @@ public boolean equals(Object other) { } @Override - public void setAttributeToNodes(Map> map) { + public void setAttributeToNodes( + Map> map) { initAttributesToNodes(); attributesToNodes.clear(); attributesToNodes.putAll(map); } @Override - public Map> getAttributesToNodes() { + public Map> getAttributesToNodes() { initAttributesToNodes(); return this.attributesToNodes; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java index 385155f3f2..41cd808d6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java @@ -17,19 +17,19 @@ */ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; -import static org.apache.hadoop.classification.InterfaceAudience.*; -import static org.apache.hadoop.classification.InterfaceStability.*; -import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; -import org.apache.hadoop.yarn.api.records.NodeAttribute; -import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; -import org.apache.hadoop.yarn.proto.YarnServiceProtos; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; - -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeInfoPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeInfoProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto; /** * Cluster node attributes response. @@ -42,7 +42,7 @@ public class GetClusterNodeAttributesResponsePBImpl private GetClusterNodeAttributesResponseProto proto = GetClusterNodeAttributesResponseProto.getDefaultInstance(); private GetClusterNodeAttributesResponseProto.Builder builder = null; - private Set updatedNodeAttributes; + private Set clusterNodeAttributes; private boolean viaProto = false; public GetClusterNodeAttributesResponsePBImpl() { @@ -72,7 +72,7 @@ private void mergeLocalToProto() { } private void mergeLocalToBuilder() { - if (this.updatedNodeAttributes != null) { + if (this.clusterNodeAttributes != null) { addNodeAttributesToProto(); } } @@ -80,11 +80,12 @@ private void mergeLocalToBuilder() { private void addNodeAttributesToProto() { maybeInitBuilder(); builder.clearNodeAttributes(); - List protoList = new ArrayList<>(); - for (NodeAttribute r : this.updatedNodeAttributes) { - protoList.add(convertToProtoFormat(r)); + if (clusterNodeAttributes == null || clusterNodeAttributes.isEmpty()) { + return; } - builder.addAllNodeAttributes(protoList); + + builder.addAllNodeAttributes(clusterNodeAttributes.stream() + .map(s -> convertToProtoFormat(s)).collect(Collectors.toSet())); } @Override @@ -112,41 +113,44 @@ private void maybeInitBuilder() { } @Override - public synchronized void setNodeAttributes(Set attributes) { + public synchronized void setNodeAttributes( + Set attributes) { maybeInitBuilder(); - this.updatedNodeAttributes = new HashSet<>(); + this.clusterNodeAttributes = new HashSet<>(); if (attributes == null) { builder.clearNodeAttributes(); return; } - this.updatedNodeAttributes.addAll(attributes); + this.clusterNodeAttributes.addAll(attributes); } @Override - public synchronized Set getNodeAttributes() { - if (this.updatedNodeAttributes != null) { - return this.updatedNodeAttributes; + public synchronized Set getNodeAttributes() { + if (this.clusterNodeAttributes != null) { + return this.clusterNodeAttributes; } initLocalNodeAttributes(); - return this.updatedNodeAttributes; + return this.clusterNodeAttributes; } private void initLocalNodeAttributes() { YarnServiceProtos.GetClusterNodeAttributesResponseProtoOrBuilder p = viaProto ? proto : builder; - List attributesProtoList = p.getNodeAttributesList(); - this.updatedNodeAttributes = new HashSet<>(); - for (NodeAttributeProto r : attributesProtoList) { - this.updatedNodeAttributes.add(convertFromProtoFormat(r)); - } + List attributesProtoList = + p.getNodeAttributesList(); + this.clusterNodeAttributes = new HashSet<>(); + clusterNodeAttributes.addAll(attributesProtoList.stream() + .map(attr -> convertFromProtoFormat(attr)).collect(Collectors.toSet())); } - private NodeAttribute convertFromProtoFormat(NodeAttributeProto p) { - return new NodeAttributePBImpl(p); + private NodeAttributeInfoProto convertToProtoFormat( + NodeAttributeInfo attributeInfo) { + return ((NodeAttributeInfoPBImpl)attributeInfo).getProto(); } - private NodeAttributeProto convertToProtoFormat(NodeAttribute t) { - return ((NodeAttributePBImpl) t).getProto(); + private NodeAttributeInfo convertFromProtoFormat( + NodeAttributeInfoProto nodeAttributeInfoProto) { + return new NodeAttributeInfoPBImpl(nodeAttributeInfoProto); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributeInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributeInfoPBImpl.java new file mode 100644 index 0000000000..bff6335f34 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributeInfoPBImpl.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; +import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeInfoProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeInfoProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeTypeProto; + +/** + * Implementation for NodeAttributeInfo. + * + */ +public class NodeAttributeInfoPBImpl extends NodeAttributeInfo { + private NodeAttributeInfoProto proto = + NodeAttributeInfoProto.getDefaultInstance(); + private NodeAttributeInfoProto.Builder builder = null; + private boolean viaProto = false; + + public NodeAttributeInfoPBImpl() { + builder = NodeAttributeInfoProto.newBuilder(); + } + + public NodeAttributeInfoPBImpl(NodeAttributeInfoProto proto) { + this.proto = proto; + viaProto = true; + } + + public NodeAttributeInfoProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = NodeAttributeInfoProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public NodeAttributeKey getAttributeKey() { + NodeAttributeInfoProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasAttributeKey()) { + return null; + } + return convertFromProtoFormat(p.getAttributeKey()); + } + + @Override + public void setAttributeKey(NodeAttributeKey attributeKey) { + maybeInitBuilder(); + if (attributeKey == null) { + builder.clearAttributeKey(); + return; + } + builder.setAttributeKey(convertToProtoFormat(attributeKey)); + } + + @Override + public NodeAttributeType getAttributeType() { + NodeAttributeInfoProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasAttributeType()) { + return null; + } + return convertFromProtoFormat(p.getAttributeType()); + } + + @Override + public void setAttributeType(NodeAttributeType attributeType) { + maybeInitBuilder(); + if (attributeType == null) { + builder.clearAttributeType(); + return; + } + builder.setAttributeType(convertToProtoFormat(attributeType)); + } + + private NodeAttributeTypeProto convertToProtoFormat( + NodeAttributeType attributeType) { + return NodeAttributeTypeProto.valueOf(attributeType.name()); + } + + private NodeAttributeType convertFromProtoFormat( + NodeAttributeTypeProto containerState) { + return NodeAttributeType.valueOf(containerState.name()); + } + + private NodeAttributeKeyPBImpl convertFromProtoFormat( + NodeAttributeKeyProto attributeKeyProto) { + return new NodeAttributeKeyPBImpl(attributeKeyProto); + } + + private NodeAttributeKeyProto convertToProtoFormat( + NodeAttributeKey attributeKey) { + return ((NodeAttributeKeyPBImpl) attributeKey).getProto(); + } + + @Override + public int hashCode() { + return getAttributeKey().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (obj instanceof NodeAttributeInfo) { + NodeAttributeInfo other = (NodeAttributeInfo) obj; + getAttributeKey().equals(other.getAttributeKey()); + return true; + } + return false; + } + + @Override + public String toString() { + return getAttributeKey().toString() + ":Type-" + getAttributeType(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributeKeyPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributeKeyPBImpl.java new file mode 100644 index 0000000000..921e767c00 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributeKeyPBImpl.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProtoOrBuilder; + +/** + * Implementation for NodeAttributeKey. + * + */ +@Private +@Unstable +public class NodeAttributeKeyPBImpl extends NodeAttributeKey { + private NodeAttributeKeyProto proto = + NodeAttributeKeyProto.getDefaultInstance(); + private NodeAttributeKeyProto.Builder builder = null; + private boolean viaProto = false; + + public NodeAttributeKeyPBImpl() { + builder = NodeAttributeKeyProto.newBuilder(); + } + + public NodeAttributeKeyPBImpl(NodeAttributeKeyProto proto) { + this.proto = proto; + viaProto = true; + } + + public NodeAttributeKeyProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = NodeAttributeKeyProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String getAttributePrefix() { + NodeAttributeKeyProtoOrBuilder p = viaProto ? proto : builder; + return p.getAttributePrefix(); + } + + @Override + public void setAttributePrefix(String attributePrefix) { + maybeInitBuilder(); + if (attributePrefix == null) { + builder.clearAttributePrefix(); + return; + } + builder.setAttributePrefix(attributePrefix); + } + + @Override + public String getAttributeName() { + NodeAttributeKeyProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasAttributeName()) { + return null; + } + return p.getAttributeName(); + } + + @Override + public void setAttributeName(String attributeName) { + maybeInitBuilder(); + if (attributeName == null) { + builder.clearAttributeName(); + return; + } + builder.setAttributeName(attributeName); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((getAttributePrefix() == null) ? 0 + : getAttributePrefix().hashCode()); + result = prime * result + + ((getAttributeName() == null) ? 0 : getAttributeName().hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (obj instanceof NodeAttributeKey) { + NodeAttributeKey other = (NodeAttributeKey) obj; + if (!compare(getAttributePrefix(), other.getAttributePrefix())) { + return false; + } + if (!compare(getAttributeName(), other.getAttributeName())) { + return false; + } + return true; + } + return false; + } + + private static boolean compare(Object left, Object right) { + if (left == null) { + return right == null; + } else { + return left.equals(right); + } + } + + @Override + public String toString() { + return "Prefix-" + getAttributePrefix() + " :Name-" + getAttributeName(); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java index 12a0eccc75..000256706c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java @@ -18,12 +18,21 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeTypeProto; +/** + * Implementation for NodeAttribute. + */ +@Private +@Unstable public class NodeAttributePBImpl extends NodeAttribute { private NodeAttributeProto proto = NodeAttributeProto.getDefaultInstance(); private NodeAttributeProto.Builder builder = null; @@ -52,22 +61,22 @@ private void maybeInitBuilder() { } @Override - public String getAttributeName() { + public NodeAttributeKey getAttributeKey() { NodeAttributeProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasAttributeName()) { + if (!p.hasAttributeKey()) { return null; } - return p.getAttributeName(); + return convertFromProtoFormat(p.getAttributeKey()); } @Override - public void setAttributeName(String attributeName) { + public void setAttributeKey(NodeAttributeKey attributeKey) { maybeInitBuilder(); - if(attributeName == null) { - builder.clearAttributeName(); + if(attributeKey == null) { + builder.clearAttributeKey(); return; } - builder.setAttributeName(attributeName); + builder.setAttributeKey(convertToProtoFormat(attributeKey)); } @Override @@ -118,15 +127,19 @@ private NodeAttributeType convertFromProtoFormat( return NodeAttributeType.valueOf(containerState.name()); } + private NodeAttributeKeyPBImpl convertFromProtoFormat( + NodeAttributeKeyProto attributeKeyProto) { + return new NodeAttributeKeyPBImpl(attributeKeyProto); + } + + private NodeAttributeKeyProto convertToProtoFormat( + NodeAttributeKey attributeKey) { + return ((NodeAttributeKeyPBImpl)attributeKey).getProto(); + } + @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((getAttributePrefix() == null) ? 0 - : getAttributePrefix().hashCode()); - result = prime * result - + ((getAttributeName() == null) ? 0 : getAttributeName().hashCode()); - return result; + return getAttributeKey().hashCode(); } @Override @@ -139,47 +152,15 @@ public boolean equals(Object obj) { } if (obj instanceof NodeAttribute) { NodeAttribute other = (NodeAttribute) obj; - if (!compare(getAttributePrefix(), other.getAttributePrefix())) { - return false; - } - if (!compare(getAttributeName(), other.getAttributeName())) { - return false; - } + getAttributeKey().equals(other.getAttributeKey()); return true; } return false; } - private static boolean compare(Object left, Object right) { - if (left == null) { - return right == null; - } else { - return left.equals(right); - } - } - - @Override - public String getAttributePrefix() { - NodeAttributeProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasAttributePrefix()) { - return null; - } - return p.getAttributePrefix(); - } - - @Override - public void setAttributePrefix(String attributePrefix) { - maybeInitBuilder(); - if(attributePrefix == null) { - builder.clearAttributePrefix(); - return; - } - builder.setAttributePrefix(attributePrefix); - } - @Override public String toString() { - return "Prefix-" + getAttributePrefix() + " :Name-" + getAttributeName() - + ":Value-" + getAttributeValue() + ":Type-" + getAttributeType(); + return getAttributeKey().toString() + ":Value-" + getAttributeValue() + + ":Type-" + getAttributeType(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeToAttributeValuePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeToAttributeValuePBImpl.java new file mode 100644 index 0000000000..7a79876f29 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeToAttributeValuePBImpl.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributeValueProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributeValueProtoOrBuilder; + +/** + * PB Implementation for NodeToAttributeValue. + * + */ +public class NodeToAttributeValuePBImpl extends NodeToAttributeValue { + private NodeToAttributeValueProto proto = + NodeToAttributeValueProto.getDefaultInstance(); + private NodeToAttributeValueProto.Builder builder = null; + private boolean viaProto = false; + + public NodeToAttributeValuePBImpl() { + builder = NodeToAttributeValueProto.newBuilder(); + } + + public NodeToAttributeValuePBImpl(NodeToAttributeValueProto proto) { + this.proto = proto; + viaProto = true; + } + + public NodeToAttributeValueProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = NodeToAttributeValueProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String getAttributeValue() { + NodeToAttributeValueProtoOrBuilder p = viaProto ? proto : builder; + return p.getAttributeValue(); + } + + @Override + public void setAttributeValue(String attributeValue) { + maybeInitBuilder(); + if (attributeValue == null) { + builder.clearAttributeValue(); + return; + } + builder.setAttributeValue(attributeValue); + } + + @Override + public String getHostname() { + NodeToAttributeValueProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasHostname()) { + return null; + } + return p.getHostname(); + } + + @Override + public void setHostname(String hostname) { + maybeInitBuilder(); + if (hostname == null) { + builder.clearHostname(); + return; + } + builder.setHostname(hostname); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((getAttributeValue() == null) ? 0 : getAttributeValue().hashCode()); + result = prime * result + + ((getHostname() == null) ? 0 : getHostname().hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (obj instanceof NodeToAttributeValue) { + NodeToAttributeValue other = (NodeToAttributeValue) obj; + if (!compare(getAttributeValue(), other.getAttributeValue())) { + return false; + } + if (!compare(getHostname(), other.getHostname())) { + return false; + } + return true; + } + return false; + } + + private static boolean compare(Object left, Object right) { + if (left == null) { + return right == null; + } else { + return left.equals(right); + } + } + + @Override + public String toString() { + return "Name-" + getHostname() + " : Attribute Value-" + + getAttributeValue(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java index 79c53e2119..68c6ec6542 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java @@ -25,6 +25,7 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; /** @@ -81,20 +82,21 @@ public abstract void removeNodeAttributes( * * @param prefix set of prefix string's for which the attributes needs to * returned - * @return set of node Attributes + * @return Set of node Attributes */ public abstract Set getClusterNodeAttributes( Set prefix); /** - * Given a attribute set, return what all Nodes have attribute mapped to it. - * If the attributes set is null or empty, all attributes mapping are - * returned. + * Return a map of Nodes to attribute value for the given NodeAttributeKeys. + * If the attributeKeys set is null or empty, then mapping for all attributes + * are returned. * - * @return a Map of attributes to set of hostnames. + * @return a Map of attributeKeys to a map of hostnames to its attribute + * values. */ - public abstract Map> getAttributesToNodes( - Set attributes); + public abstract Map> getAttributesToNodes( + Set attributes); /** * NodeAttribute to AttributeValue Map. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java index 93a27a9ebf..0dd0755e51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java @@ -19,6 +19,7 @@ import com.google.common.base.Strings; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; import java.io.IOException; import java.util.Set; @@ -115,14 +116,18 @@ public static void validateNodeAttributes(Set attributeSet) throws IOException { if (attributeSet != null && !attributeSet.isEmpty()) { for (NodeAttribute nodeAttribute : attributeSet) { - String prefix = nodeAttribute.getAttributePrefix(); + NodeAttributeKey attributeKey = nodeAttribute.getAttributeKey(); + if (attributeKey == null) { + throw new IOException("AttributeKey must be set"); + } + String prefix = attributeKey.getAttributePrefix(); if (Strings.isNullOrEmpty(prefix)) { throw new IOException("Attribute prefix must be set"); } // Verify attribute prefix format. checkAndThrowAttributePrefix(prefix); // Verify attribute name format. - checkAndThrowLabelName(nodeAttribute.getAttributeName()); + checkAndThrowLabelName(attributeKey.getAttributeName()); } } } @@ -140,8 +145,9 @@ public static Set filterAttributesByPrefix( if (Strings.isNullOrEmpty(prefix)) { return attributeSet; } - return attributeSet.stream().filter( - nodeAttribute -> prefix.equals(nodeAttribute.getAttributePrefix())) + return attributeSet.stream() + .filter(nodeAttribute -> prefix + .equals(nodeAttribute.getAttributeKey().getAttributePrefix())) .collect(Collectors.toSet()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeAttribute.java index 3b2bd16268..ffe36c8c4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeAttribute.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeAttribute.java @@ -18,8 +18,8 @@ package org.apache.hadoop.yarn.nodelabels; -import java.util.HashSet; -import java.util.Set; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeAttributeType; @@ -32,11 +32,11 @@ public class RMNodeAttribute extends AbstractLabel { private NodeAttribute attribute; // TODO need to revisit whether we need to make this concurrent implementation - private Set nodes = new HashSet<>(); + private Map nodes = new HashMap<>(); public RMNodeAttribute(NodeAttribute attribute) { - this(attribute.getAttributeName(), Resource.newInstance(0, 0), 0, - attribute); + this(attribute.getAttributeKey().getAttributeName(), + Resource.newInstance(0, 0), 0, attribute); } public RMNodeAttribute(String labelName, Resource res, int activeNMs, @@ -57,16 +57,16 @@ public NodeAttributeType getAttributeType() { return attribute.getAttributeType(); } - public void addNode(String node) { - nodes.add(node); + public void addNode(String node, AttributeValue attributeValue) { + nodes.put(node, attributeValue); } public void removeNode(String node) { nodes.remove(node); } - public Set getAssociatedNodeIds() { - return new HashSet(nodes); + public Map getAssociatedNodeIds() { + return new HashMap(nodes); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 9f3e9250e8..682d6ec68e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.api; import java.io.IOException; +import java.util.Arrays; import org.apache.commons.lang3.Range; import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; @@ -112,7 +113,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; -import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -120,6 +120,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -131,9 +132,12 @@ import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.PreemptionContainer; import org.apache.hadoop.yarn.api.records.PreemptionContract; import org.apache.hadoop.yarn.api.records.PreemptionMessage; @@ -152,8 +156,8 @@ import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceAllocationRequest; -import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.ResourceSizing; @@ -183,10 +187,13 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ExecutionTypeRequestPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeKeyPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeInfoPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeToAttributeValuePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionContainerPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionContractPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl; @@ -222,10 +229,14 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeInfoProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributeValueProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributesProto; import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionContractProto; import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionMessageProto; @@ -243,7 +254,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.StrictPreemptionContractProto; import org.apache.hadoop.yarn.proto.YarnProtos.URLProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnClusterMetricsProto; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributesProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto; @@ -274,6 +284,7 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsRequestProto; @@ -304,6 +315,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProto; @@ -328,9 +341,6 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; @@ -366,7 +376,6 @@ import org.junit.Test; import com.google.common.collect.ImmutableSet; -import java.util.Arrays; /** * Test class for YARN API protocol records. @@ -450,10 +459,12 @@ public static void setup() throws Exception { generateByNewInstance(SchedulingRequest.class); generateByNewInstance(RejectedSchedulingRequest.class); //for Node attribute support + generateByNewInstance(NodeAttributeKey.class); generateByNewInstance(NodeAttribute.class); generateByNewInstance(NodeToAttributes.class); + generateByNewInstance(NodeToAttributeValue.class); + generateByNewInstance(NodeAttributeInfo.class); generateByNewInstance(NodesToAttributesMappingRequest.class); - } @Test @@ -1249,11 +1260,29 @@ public void testGetAllResourceTypesInfoResponsePBImpl() throws Exception { YarnServiceProtos.GetAllResourceTypeInfoResponseProto.class); } + @Test + public void testNodeAttributeKeyPBImpl() throws Exception { + validatePBImplRecord(NodeAttributeKeyPBImpl.class, + NodeAttributeKeyProto.class); + } + + @Test + public void testNodeToAttributeValuePBImpl() throws Exception { + validatePBImplRecord(NodeToAttributeValuePBImpl.class, + NodeToAttributeValueProto.class); + } + @Test public void testNodeAttributePBImpl() throws Exception { validatePBImplRecord(NodeAttributePBImpl.class, NodeAttributeProto.class); } + @Test + public void testNodeAttributeInfoPBImpl() throws Exception { + validatePBImplRecord(NodeAttributeInfoPBImpl.class, + NodeAttributeInfoProto.class); + } + @Test public void testNodeToAttributesPBImpl() throws Exception { validatePBImplRecord(NodeToAttributesPBImpl.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java index d4384b47aa..bad74d80db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; import org.apache.hadoop.yarn.api.records.NodeAttributeType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.BeforeClass; @@ -120,7 +121,8 @@ public void testNodeAttributesFetchInterval() while(times>0) { Set current = spyProvider.getDescriptors(); Assert.assertEquals(1, current.size()); - String attributeName = current.iterator().next().getAttributeName(); + String attributeName = + current.iterator().next().getAttributeKey().getAttributeName(); if ("host".equals(attributeName)){ numOfOldValue++; } else if ("os".equals(attributeName)) { @@ -173,7 +175,7 @@ public void testDisableFetchNodeAttributes() throws IOException, GenericTestUtils.waitFor(() -> { Set attributes = spyProvider.getDescriptors(); return "os".equalsIgnoreCase(attributes - .iterator().next().getAttributeName()); + .iterator().next().getAttributeKey().getAttributeName()); }, 500, 1000); } catch (Exception e) { // Make sure we get the timeout exception. @@ -204,21 +206,22 @@ public void testParseConfiguration() throws IOException { Iterator ait = attributes.iterator(); while(ait.hasNext()) { - NodeAttribute at = ait.next(); + NodeAttribute attr = ait.next(); + NodeAttributeKey at = attr.getAttributeKey(); if (at.getAttributeName().equals("hostname")) { Assert.assertEquals("hostname", at.getAttributeName()); Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED, at.getAttributePrefix()); Assert.assertEquals(NodeAttributeType.STRING, - at.getAttributeType()); - Assert.assertEquals("host1234", at.getAttributeValue()); + attr.getAttributeType()); + Assert.assertEquals("host1234", attr.getAttributeValue()); } else if (at.getAttributeName().equals("uptime")) { Assert.assertEquals("uptime", at.getAttributeName()); Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED, at.getAttributePrefix()); Assert.assertEquals(NodeAttributeType.STRING, - at.getAttributeType()); - Assert.assertEquals("321543", at.getAttributeValue()); + attr.getAttributeType()); + Assert.assertEquals("321543", attr.getAttributeValue()); } else { Assert.fail("Unexpected attribute"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java index f764626b4e..3e2e161cfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java @@ -129,7 +129,7 @@ public void testNodeAttributeScriptProvider() .getDescriptors().iterator(); while (it.hasNext()) { NodeAttribute att = it.next(); - switch (att.getAttributeName()) { + switch (att.getAttributeKey().getAttributeName()) { case "host": Assert.assertEquals(NodeAttributeType.STRING, att.getAttributeType()); Assert.assertEquals("host1234", att.getAttributeValue()); @@ -143,7 +143,8 @@ public void testNodeAttributeScriptProvider() Assert.assertEquals("10.0.0.1", att.getAttributeValue()); break; default: - Assert.fail("Unexpected attribute name " + att.getAttributeName()); + Assert.fail("Unexpected attribute name " + + att.getAttributeKey().getAttributeName()); break; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 12f8aaf5df..77f8f6615f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -1035,7 +1035,7 @@ private Map> validateAndFetch( List nodeAttributes = nodeToAttributes.getNodeAttributes(); if (!nodeAttributes.stream() .allMatch(nodeAttribute -> NodeAttribute.PREFIX_CENTRALIZED - .equals(nodeAttribute.getAttributePrefix()))) { + .equals(nodeAttribute.getAttributeKey().getAttributePrefix()))) { throw new IOException("Invalid Attribute Mapping for the node " + node + ". Prefix should be " + NodeAttribute.PREFIX_CENTRALIZED); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 3f2435549b..ad796f6307 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -29,12 +29,14 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import org.apache.commons.cli.UnrecognizedOptionException; import org.apache.commons.lang3.Range; @@ -134,8 +136,11 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -155,6 +160,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.nodelabels.AttributeValue; import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; @@ -1851,9 +1857,23 @@ public GetAttributesToNodesResponse getAttributesToNodes( GetAttributesToNodesRequest request) throws YarnException, IOException { NodeAttributesManager attributesManager = rmContext.getNodeAttributesManager(); - GetAttributesToNodesResponse response = GetAttributesToNodesResponse - .newInstance(attributesManager - .getAttributesToNodes(request.getNodeAttributes())); + Map> attrToNodesWithStrVal = + new HashMap<>(); + Map> attributesToNodes = + attributesManager.getAttributesToNodes(request.getNodeAttributes()); + for (Map.Entry> attrib : + attributesToNodes.entrySet()) { + Map nodesToVal = attrib.getValue(); + List nodeToAttrValList = new ArrayList<>(); + for (Map.Entry nodeToVal : nodesToVal + .entrySet()) { + nodeToAttrValList.add(NodeToAttributeValue + .newInstance(nodeToVal.getKey(), nodeToVal.getValue().getValue())); + } + attrToNodesWithStrVal.put(attrib.getKey(), nodeToAttrValList); + } + GetAttributesToNodesResponse response = + GetAttributesToNodesResponse.newInstance(attrToNodesWithStrVal); return response; } @@ -1865,8 +1885,11 @@ public GetClusterNodeAttributesResponse getClusterNodeAttributes( rmContext.getNodeAttributesManager(); Set attributes = attributesManager.getClusterNodeAttributes(null); + GetClusterNodeAttributesResponse response = - GetClusterNodeAttributesResponse.newInstance(attributes); + GetClusterNodeAttributesResponse.newInstance( + attributes.stream().map(attr -> NodeAttributeInfo.newInstance(attr)) + .collect(Collectors.toSet())); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 4f4400fcc4..cbb5ecf4c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -660,7 +660,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) // Validate attributes if (!nodeAttributes.stream().allMatch( nodeAttribute -> NodeAttribute.PREFIX_DISTRIBUTED - .equals(nodeAttribute.getAttributePrefix()))) { + .equals(nodeAttribute.getAttributeKey().getAttributePrefix()))) { // All attributes must be in same prefix: nm.yarn.io. // Since we have the checks in NM to make sure attributes reported // in HB are with correct prefix, so it should not reach here. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java index 16456024d9..6b176d6d52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java @@ -19,21 +19,20 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentHashMap.KeySetView; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; -import java.util.ArrayList; -import java.util.List; import com.google.common.base.Strings; import org.apache.commons.lang3.StringUtils; @@ -42,6 +41,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; +import com.google.common.base.Strings; + /** * Manager holding the attributes to Labels. */ @@ -75,8 +77,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { // TODO may be we can have a better collection here. // this will be updated to get the attributeName to NM mapping - private ConcurrentHashMap clusterAttributes = - new ConcurrentHashMap<>(); + private ConcurrentHashMap clusterAttributes + = new ConcurrentHashMap<>(); // hostname -> (Map (attributeName -> NodeAttribute)) // Instead of NodeAttribute, plan to have it in future as AttributeValue @@ -149,7 +151,7 @@ private NodeAttributeStore getAttributeStoreClass(Configuration conf) { private void internalUpdateAttributesOnNodes( Map> nodeAttributeMapping, AttributeMappingOperationType op, - Map newAttributesToBeAdded, + Map newAttributesToBeAdded, String attributePrefix) { try { writeLock.lock(); @@ -210,13 +212,14 @@ private void internalUpdateAttributesOnNodes( private void removeNodeFromAttributes(String nodeHost, Set attributeMappings) { for (NodeAttribute rmAttribute : attributeMappings) { - RMNodeAttribute host = clusterAttributes.get(rmAttribute); + RMNodeAttribute host = + clusterAttributes.get(rmAttribute.getAttributeKey()); if (host != null) { host.removeNode(nodeHost); // If there is no other host has such attribute, // remove it from the global mapping. if (host.getAssociatedNodeIds().isEmpty()) { - clusterAttributes.remove(rmAttribute); + clusterAttributes.remove(rmAttribute.getAttributeKey()); } } } @@ -224,12 +227,16 @@ private void removeNodeFromAttributes(String nodeHost, private void addNodeToAttribute(String nodeHost, Map attributeMappings) { - for (NodeAttribute attribute : attributeMappings.keySet()) { - RMNodeAttribute rmNodeAttribute = clusterAttributes.get(attribute); + for (Entry attributeEntry : attributeMappings + .entrySet()) { + + RMNodeAttribute rmNodeAttribute = + clusterAttributes.get(attributeEntry.getKey().getAttributeKey()); if (rmNodeAttribute != null) { - rmNodeAttribute.addNode(nodeHost); + rmNodeAttribute.addNode(nodeHost, attributeEntry.getValue()); } else { - clusterAttributes.put(attribute, new RMNodeAttribute(attribute)); + clusterAttributes.put(attributeEntry.getKey().getAttributeKey(), + new RMNodeAttribute(attributeEntry.getKey())); } } } @@ -257,7 +264,7 @@ private void replaceNodeToAttribute(String nodeHost, String prefix, */ protected Map> validate( Map> nodeAttributeMapping, - Map newAttributesToBeAdded, + Map newAttributesToBeAdded, boolean isRemoveOperation) throws IOException { Map> nodeToAttributesMap = new TreeMap<>(); @@ -274,19 +281,21 @@ protected Map> validate( // validate for attributes for (NodeAttribute attribute : nodeToAttrMappingEntry.getValue()) { - String attributeName = attribute.getAttributeName().trim(); + NodeAttributeKey attributeKey = attribute.getAttributeKey(); + String attributeName = attributeKey.getAttributeName().trim(); NodeLabelUtil.checkAndThrowLabelName(attributeName); NodeLabelUtil - .checkAndThrowAttributePrefix(attribute.getAttributePrefix()); + .checkAndThrowAttributePrefix(attributeKey.getAttributePrefix()); // ensure trimmed values are set back - attribute.setAttributeName(attributeName); - attribute.setAttributePrefix(attribute.getAttributePrefix().trim()); + attributeKey.setAttributeName(attributeName); + attributeKey + .setAttributePrefix(attributeKey.getAttributePrefix().trim()); // verify for type against prefix/attributeName if (validateForAttributeTypeMismatch(isRemoveOperation, attribute, newAttributesToBeAdded)) { - newAttributesToBeAdded.put(attribute, + newAttributesToBeAdded.put(attribute.getAttributeKey(), new RMNodeAttribute(attribute)); } // TODO type based value setting needs to be done using a factory @@ -310,9 +319,11 @@ protected Map> validate( */ private boolean validateForAttributeTypeMismatch(boolean isRemoveOperation, NodeAttribute attribute, - Map newAttributes) + Map newAttributes) throws IOException { - if (isRemoveOperation && !clusterAttributes.containsKey(attribute)) { + NodeAttributeKey attributeKey = attribute.getAttributeKey(); + if (isRemoveOperation + && !clusterAttributes.containsKey(attributeKey)) { // no need to validate anything as its remove operation and attribute // doesn't exist. return false; // no need to add as its remove operation @@ -320,10 +331,10 @@ private boolean validateForAttributeTypeMismatch(boolean isRemoveOperation, // already existing or attribute is mapped to another Node in the // current command, then check whether the attribute type is matching NodeAttribute existingAttribute = - (clusterAttributes.containsKey((attribute)) - ? clusterAttributes.get(attribute).getAttribute() - : (newAttributes.containsKey(attribute) - ? newAttributes.get(attribute).getAttribute() + (clusterAttributes.containsKey(attributeKey) + ? clusterAttributes.get(attributeKey).getAttribute() + : (newAttributes.containsKey(attributeKey) + ? newAttributes.get(attributeKey).getAttribute() : null)); if (existingAttribute == null) { return true; @@ -331,7 +342,7 @@ private boolean validateForAttributeTypeMismatch(boolean isRemoveOperation, .getAttributeType()) { throw new IOException("Attribute name - type is not matching with " + "already configured mapping for the attribute " - + attribute.getAttributeName() + " existing : " + + attributeKey + " existing : " + existingAttribute.getAttributeType() + ", new :" + attribute.getAttributeType()); } @@ -347,37 +358,39 @@ protected String normalizeAttributeValue(String value) { } @Override - public Set getClusterNodeAttributes(Set prefix) { + public Set getClusterNodeAttributes( + Set prefix) { Set attributes = new HashSet<>(); - KeySetView allAttributes = - clusterAttributes.keySet(); + Set> allAttributes = + clusterAttributes.entrySet(); // Return all if prefix is not given. - if (prefix == null || prefix.isEmpty()) { - attributes.addAll(allAttributes); - return attributes; - } + boolean forAllPrefix = prefix == null || prefix.isEmpty(); // Try search attributes by prefix and return valid ones. - Iterator iterator = allAttributes.iterator(); + Iterator> iterator = + allAttributes.iterator(); while (iterator.hasNext()) { - NodeAttribute current = iterator.next(); - if (prefix.contains(current.getAttributePrefix())) { - attributes.add(current); + Entry current = iterator.next(); + NodeAttributeKey attrID = current.getKey(); + RMNodeAttribute rmAttr = current.getValue(); + if (forAllPrefix || prefix.contains(attrID.getAttributePrefix())) { + attributes.add(rmAttr.getAttribute()); } } return attributes; } @Override - public Map> getAttributesToNodes( - Set attributes) { + public Map> getAttributesToNodes( + Set attributes) { try { readLock.lock(); boolean fetchAllAttributes = (attributes == null || attributes.isEmpty()); - Map> attributesToNodes = new HashMap<>(); - for (Entry attributeEntry : + Map> attributesToNodes = + new HashMap<>(); + for (Entry attributeEntry : clusterAttributes.entrySet()) { - if (fetchAllAttributes || attributes - .contains(attributeEntry.getKey())) { + if (fetchAllAttributes + || attributes.contains(attributeEntry.getKey())) { attributesToNodes.put(attributeEntry.getKey(), attributeEntry.getValue().getAssociatedNodeIds()); } @@ -391,8 +404,8 @@ public Map> getAttributesToNodes( public Resource getResourceByAttribute(NodeAttribute attribute) { try { readLock.lock(); - return clusterAttributes.containsKey(attribute) - ? clusterAttributes.get(attribute).getResource() + return clusterAttributes.containsKey(attribute.getAttributeKey()) + ? clusterAttributes.get(attribute.getAttributeKey()).getResource() : Resource.newInstance(0, 0); } finally { readLock.unlock(); @@ -425,7 +438,8 @@ public List getNodeToAttributes(Set prefix) { attrs = new ArrayList<>(); for (Entry nodeAttr : v.attributes .entrySet()) { - if (prefix.contains(nodeAttr.getKey().getAttributePrefix())) { + if (prefix.contains( + nodeAttr.getKey().getAttributeKey().getAttributePrefix())) { attrs.add(nodeAttr.getKey()); } } @@ -473,7 +487,7 @@ public void activateNode(NodeId nodeId, Resource resource) { } host.activateNode(resource); for (NodeAttribute attribute : host.getAttributes().keySet()) { - clusterAttributes.get(attribute).removeNode(resource); + clusterAttributes.get(attribute.getAttributeKey()).removeNode(resource); } } finally { writeLock.unlock(); @@ -485,7 +499,8 @@ public void deactivateNode(NodeId nodeId) { writeLock.lock(); Host host = nodeCollections.get(nodeId.getHost()); for (NodeAttribute attribute : host.getAttributes().keySet()) { - clusterAttributes.get(attribute).removeNode(host.getResource()); + clusterAttributes.get(attribute.getAttributeKey()) + .removeNode(host.getResource()); } host.deactivateNode(); } finally { @@ -531,7 +546,8 @@ public void replaceAttributes( this.attributes.entrySet().iterator(); while (it.hasNext()) { Entry current = it.next(); - if (prefix.equals(current.getKey().getAttributePrefix())) { + if (prefix.equals( + current.getKey().getAttributeKey().getAttributePrefix())) { it.remove(); } } @@ -659,7 +675,7 @@ private void processMapping( Map> nodeAttributeMapping, AttributeMappingOperationType mappingType, String attributePrefix) throws IOException { - Map newAttributesToBeAdded = + Map newAttributesToBeAdded = new HashMap<>(); Map> validMapping = validate(nodeAttributeMapping, newAttributesToBeAdded, false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java index 1645d13836..93b901ec49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java @@ -21,9 +21,11 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeLabel; /** @@ -56,4 +58,21 @@ public static void verifyCentralizedNodeLabelConfEnabled(String operation, throw new IOException(msg); } } + + /** + * Returns a set of node attributes whose name exists in the provided + * attributeNames list. + * + * @param attributeNames For this given list of attribute names get the + * cluster NodeAttributes + * @param clusterNodeAttributes set of node Attributes + * @return set of Node Attributes which maps to the give attributes names + */ + public static Set getNodeAttributesByName( + Set attributeNames, Set clusterNodeAttributes) { + return clusterNodeAttributes.stream() + .filter(attribute -> attributeNames + .contains(attribute.getAttributeKey().getAttributeName())) + .collect(Collectors.toSet()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java index bbc2ec3145..838431216a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java @@ -41,8 +41,8 @@ public NodeAttributeInfo() { } public NodeAttributeInfo(NodeAttribute nodeAttribute) { - this.prefix = nodeAttribute.getAttributePrefix(); - this.name = nodeAttribute.getAttributeName(); + this.prefix = nodeAttribute.getAttributeKey().getAttributePrefix(); + this.name = nodeAttribute.getAttributeKey().getAttributeName(); this.type = nodeAttribute.getAttributeType().toString(); this.value = nodeAttribute.getAttributeValue(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 95ad35aad0..39892f2277 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -18,16 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; -import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; - -import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; -import org.apache.hadoop.yarn.api.records.NodeAttribute; -import org.apache.hadoop.yarn.api.records.NodeAttributeType; -import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -78,6 +68,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -88,6 +82,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -117,10 +113,15 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueConfigurations; @@ -142,6 +143,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -163,7 +165,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; - import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -176,11 +177,11 @@ import org.junit.Assert; import org.junit.Assume; import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; public class TestClientRMService { @@ -2048,11 +2049,12 @@ protected ClientRMService createClientRMService() { GetClusterNodeAttributesRequest.newInstance(); GetClusterNodeAttributesResponse response = client.getClusterNodeAttributes(request); - Set attributes = response.getNodeAttributes(); + Set attributes = response.getNodeAttributes(); Assert.assertEquals("Size not correct", 3, attributes.size()); - Assert.assertTrue(attributes.contains(gpu)); - Assert.assertTrue(attributes.contains(os)); - Assert.assertTrue(attributes.contains(docker)); + Assert.assertTrue(attributes.contains(NodeAttributeInfo.newInstance(gpu))); + Assert.assertTrue(attributes.contains(NodeAttributeInfo.newInstance(os))); + Assert + .assertTrue(attributes.contains(NodeAttributeInfo.newInstance(docker))); rpc.stopProxy(client, conf); rm.close(); } @@ -2071,17 +2073,17 @@ protected ClientRMService createClientRMService() { NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager(); String node1 = "host1"; String node2 = "host2"; - NodeAttribute gpu = NodeAttribute - .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", - NodeAttributeType.STRING, "nvida"); - NodeAttribute os = NodeAttribute - .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", + NodeAttribute gpu = + NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvidia"); + NodeAttribute os = + NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", NodeAttributeType.STRING, "windows64"); - NodeAttribute docker = NodeAttribute - .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", + NodeAttribute docker = + NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", NodeAttributeType.STRING, "docker0"); - NodeAttribute dist = NodeAttribute - .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION", + NodeAttribute dist = + NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION", NodeAttributeType.STRING, "3_0_2"); Map> nodes = new HashMap<>(); nodes.put(node1, ImmutableSet.of(gpu, os, dist)); @@ -2099,35 +2101,55 @@ protected ClientRMService createClientRMService() { GetAttributesToNodesRequest.newInstance(); GetAttributesToNodesResponse response = client.getAttributesToNodes(request); - Map> attrs = response.getAttributesToNodes(); + Map> attrs = + response.getAttributesToNodes(); Assert.assertEquals(response.getAttributesToNodes().size(), 4); - Assert.assertEquals(attrs.get(dist).size(), 2); - Assert.assertEquals(attrs.get(os).size(), 1); - Assert.assertEquals(attrs.get(gpu).size(), 1); - Assert.assertTrue(attrs.get(dist).contains(node1)); - Assert.assertTrue(attrs.get(dist).contains(node2)); - Assert.assertTrue(attrs.get(docker).contains(node2)); + Assert.assertEquals(attrs.get(dist.getAttributeKey()).size(), 2); + Assert.assertEquals(attrs.get(os.getAttributeKey()).size(), 1); + Assert.assertEquals(attrs.get(gpu.getAttributeKey()).size(), 1); + Assert.assertTrue(findHostnameAndValInMapping(node1, "3_0_2", + attrs.get(dist.getAttributeKey()))); + Assert.assertTrue(findHostnameAndValInMapping(node2, "3_0_2", + attrs.get(dist.getAttributeKey()))); + Assert.assertTrue(findHostnameAndValInMapping(node2, "docker0", + attrs.get(docker.getAttributeKey()))); - GetAttributesToNodesRequest request2 = - GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker)); + GetAttributesToNodesRequest request2 = GetAttributesToNodesRequest + .newInstance(ImmutableSet.of(docker.getAttributeKey())); GetAttributesToNodesResponse response2 = client.getAttributesToNodes(request2); - Map> attrs2 = response2.getAttributesToNodes(); - Assert.assertEquals(response2.getAttributesToNodes().size(), 1); - Assert.assertTrue(attrs.get(docker).contains(node2)); + Map> attrs2 = + response2.getAttributesToNodes(); + Assert.assertEquals(attrs2.size(), 1); + Assert.assertTrue(findHostnameAndValInMapping(node2, "docker0", + attrs2.get(docker.getAttributeKey()))); GetAttributesToNodesRequest request3 = - GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker, os)); + GetAttributesToNodesRequest.newInstance( + ImmutableSet.of(docker.getAttributeKey(), os.getAttributeKey())); GetAttributesToNodesResponse response3 = client.getAttributesToNodes(request3); - Map> attrs3 = response3.getAttributesToNodes(); - Assert.assertEquals(response3.getAttributesToNodes().size(), 2); - Assert.assertTrue(attrs.get(os).contains(node1)); - Assert.assertTrue(attrs.get(docker).contains(node2)); + Map> attrs3 = + response3.getAttributesToNodes(); + Assert.assertEquals(attrs3.size(), 2); + Assert.assertTrue(findHostnameAndValInMapping(node1, "windows64", + attrs3.get(os.getAttributeKey()))); + Assert.assertTrue(findHostnameAndValInMapping(node2, "docker0", + attrs3.get(docker.getAttributeKey()))); rpc.stopProxy(client, conf); rm.close(); } + private boolean findHostnameAndValInMapping(String hostname, String attrVal, + List mappingVals) { + for (NodeToAttributeValue value : mappingVals) { + if (value.getHostname().equals(hostname)) { + return attrVal.equals(value.getAttributeValue()); + } + } + return false; + } + @Test(timeout = 120000) public void testGetNodesToAttributes() throws IOException, YarnException { MockRM rm = new MockRM() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index adb7fe0802..e40b3c051c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -880,7 +880,7 @@ public void testNodeHeartbeatWithNodeAttributes() throws Exception { .getAttributesForNode(nodeId.getHost()); Assert.assertEquals(1, attrs.size()); NodeAttribute na = attrs.keySet().iterator().next(); - Assert.assertEquals("host", na.getAttributeName()); + Assert.assertEquals("host", na.getAttributeKey().getAttributeName()); Assert.assertEquals("host2", na.getAttributeValue()); Assert.assertEquals(NodeAttributeType.STRING, na.getAttributeType()); @@ -900,7 +900,7 @@ public void testNodeHeartbeatWithNodeAttributes() throws Exception { attrs = attributeManager.getAttributesForNode(nodeId.getHost()); Assert.assertEquals(1, attrs.size()); na = attrs.keySet().iterator().next(); - Assert.assertEquals("host", na.getAttributeName()); + Assert.assertEquals("host", na.getAttributeKey().getAttributeName()); Assert.assertEquals("host3", na.getAttributeValue()); Assert.assertEquals(NodeAttributeType.STRING, na.getAttributeType()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java index e2ee8b4763..502f9d051c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java @@ -253,8 +253,10 @@ public void testRecoverFromEditLog() throws Exception { public void checkNodeAttributeEqual(NodeAttribute atr1, NodeAttribute atr2) { Assert.assertEquals(atr1.getAttributeType(), atr2.getAttributeType()); - Assert.assertEquals(atr1.getAttributeName(), atr2.getAttributeName()); - Assert.assertEquals(atr1.getAttributePrefix(), atr2.getAttributePrefix()); + Assert.assertEquals(atr1.getAttributeKey().getAttributeName(), + atr2.getAttributeKey().getAttributeName()); + Assert.assertEquals(atr1.getAttributeKey().getAttributePrefix(), + atr2.getAttributeKey().getAttributePrefix()); Assert.assertEquals(atr1.getAttributeValue(), atr2.getAttributeValue()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java index b8c5bc9e30..9bc9388fb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java @@ -344,11 +344,12 @@ public void testReplaceNodeAttributes() throws IOException { clusterAttributes = attributesManager.getClusterNodeAttributes( Sets.newHashSet(NodeAttribute.PREFIX_DISTRIBUTED)); Assert.assertEquals(1, clusterAttributes.size()); - NodeAttribute att = clusterAttributes.iterator().next(); - Assert.assertEquals("dist-node-attribute-v2_0", att.getAttributeName()); + NodeAttribute attr = clusterAttributes.iterator().next(); + Assert.assertEquals("dist-node-attribute-v2_0", + attr.getAttributeKey().getAttributeName()); Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED, - att.getAttributePrefix()); - Assert.assertEquals("dist_v3_0", att.getAttributeValue()); + attr.getAttributeKey().getAttributePrefix()); + Assert.assertEquals("dist_v3_0", attr.getAttributeValue()); // Replace all attributes toReplaceMap.put(HOSTNAMES[0], diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index ceabe6563c..3d9b3946f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -849,6 +849,6 @@ public GetClusterNodeAttributesResponse getClusterNodeAttributes( @Override public GetNodesToAttributesResponse getNodesToAttributes( GetNodesToAttributesRequest request) throws YarnException, IOException { - throw new NotImplementedException(); + throw new NotImplementedException("Code is not implemented"); } }