From 0a01b1350d69dacaf101d52e54f67c91e28ab1ee Mon Sep 17 00:00:00 2001 From: Naganarasimha Date: Tue, 10 Apr 2018 07:28:53 +0800 Subject: [PATCH] YARN-8100. Support API interface to query cluster attributes and attribute to nodes. Contributed by Bibin A Chundatt. --- .../hadoop/mapred/ResourceMgrDelegate.java | 13 ++ .../hadoop/mapred/TestClientRedirect.java | 17 ++ .../yarn/api/ApplicationClientProtocol.java | 37 ++++ .../GetAttributesToNodesRequest.java | 70 +++++++ .../GetAttributesToNodesResponse.java | 62 ++++++ .../GetClusterNodeAttributesRequest.java | 47 +++++ .../GetClusterNodeAttributesResponse.java | 72 +++++++ .../hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../proto/applicationclient_protocol.proto | 2 + .../src/main/proto/yarn_protos.proto | 4 + .../src/main/proto/yarn_service_protos.proto | 15 ++ .../hadoop/yarn/client/api/YarnClient.java | 36 +++- .../yarn/client/api/impl/YarnClientImpl.java | 21 +- ...ApplicationClientProtocolPBClientImpl.java | 37 ++++ ...pplicationClientProtocolPBServiceImpl.java | 44 +++++ .../pb/GetAttributesToNodesRequestPBImpl.java | 175 +++++++++++++++++ .../GetAttributesToNodesResponsePBImpl.java | 184 ++++++++++++++++++ ...GetClusterNodeAttributesRequestPBImpl.java | 75 +++++++ ...etClusterNodeAttributesResponsePBImpl.java | 156 +++++++++++++++ .../nodelabels/NodeAttributesManager.java | 9 +- .../hadoop/yarn/api/TestPBImplRecords.java | 28 +++ .../server/MockResourceManagerFacade.java | 17 ++ .../resourcemanager/ClientRMService.java | 30 +++ .../nodelabels/NodeAttributesManagerImpl.java | 35 ++-- .../resourcemanager/TestClientRMService.java | 126 ++++++++++++ .../DefaultClientRequestInterceptor.java | 17 ++ .../clientrm/FederationClientInterceptor.java | 17 ++ .../clientrm/RouterClientRMService.java | 19 ++ .../PassThroughClientRequestInterceptor.java | 17 ++ 29 files changed, 1362 insertions(+), 22 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index ac4b73b241..4f96a6b669 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -538,4 +539,16 @@ public List getResourceTypeInfo() throws YarnException, IOException { return client.getResourceTypeInfo(); } + + @Override + public Set getClusterAttributes() + throws YarnException, IOException { + return client.getClusterAttributes(); + } + + @Override + public Map> getAttributesToNodes( + Set attributes) throws YarnException, IOException { + return client.getAttributesToNodes(attributes); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index f97d0a48f7..23a1a853a9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -82,8 +82,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -521,6 +525,19 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo( throws YarnException, IOException { return null; } + + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + return null; + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index 3c4e4d0100..8661a78fbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -27,8 +27,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -642,4 +646,37 @@ GetResourceProfileResponse getResourceProfile( @Unstable GetAllResourceTypeInfoResponse getResourceTypeInfo( GetAllResourceTypeInfoRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by client to get attributes to nodes mappings + * available in ResourceManager. + *

+ * + * @param request request to get details of attributes to nodes mapping. + * @return Response containing the details of attributes to nodes mappings. + * @throws YarnException if any error happens inside YARN + * @throws IOException incase of other errors + */ + @Public + @Unstable + GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by client to get node attributes available in + * ResourceManager. + *

+ * + * @param request request to get node attributes collection of this cluster. + * @return Response containing node attributes collection. + * @throws YarnException if any error happens inside YARN. + * @throws IOException incase of other errors. + */ + @Public + @Unstable + GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java new file mode 100644 index 0000000000..d9531b0c38 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.util.Records; + +import java.util.Set; + +/** + *

+ * The request from clients to get attribtues to nodes mapping + * in the cluster from the ResourceManager. + *

+ * + * @see ApplicationClientProtocol#getAttributesToNodes + * (GetAttributesToNodesRequest) + */ +@Public +@Evolving +public abstract class GetAttributesToNodesRequest { + + public static GetAttributesToNodesRequest newInstance() { + return Records.newRecord(GetAttributesToNodesRequest.class); + } + + public static GetAttributesToNodesRequest newInstance( + Set attributes) { + GetAttributesToNodesRequest request = + Records.newRecord(GetAttributesToNodesRequest.class); + request.setNodeAttributes(attributes); + return request; + } + + /** + * Set node attributes for which the mapping is required. + * + * @param attributes Set provided. + */ + @Public + @Unstable + public abstract void setNodeAttributes(Set attributes); + + /** + * Get node attributes for which mapping mapping is required. + * + * @return Set + */ + @Public + @Unstable + public abstract Set getNodeAttributes(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java new file mode 100644 index 0000000000..4fdb1f70a9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import static org.apache.hadoop.classification.InterfaceAudience.Public; +import static org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.util.Records; + +import java.util.Map; +import java.util.Set; + +/** + *

+ * The response sent by the ResourceManager to a client requesting + * attributes to hostname mapping. + *

+ * + * @see ApplicationClientProtocol#getAttributesToNodes + * (GetAttributesToNodesRequest) + */ +@Public +@Evolving +public abstract class GetAttributesToNodesResponse { + public static GetAttributesToNodesResponse newInstance( + Map> map) { + GetAttributesToNodesResponse response = + Records.newRecord(GetAttributesToNodesResponse.class); + response.setAttributeToNodes(map); + return response; + } + + @Public + @Evolving + public abstract void setAttributeToNodes(Map> map); + + /* + * Get attributes to node hostname mapping. + * + * @return Map> node attributes to hostname + * mapping. + */ + @Public + @Evolving + public abstract Map> getAttributesToNodes(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java new file mode 100644 index 0000000000..ca81f9a084 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The request from clients to get node attributes in the cluster from the + * ResourceManager. + *

+ * + * @see ApplicationClientProtocol#getClusterNodeAttributes + * (GetClusterNodeAttributesRequest) + */ +@Public +@Evolving +public abstract class GetClusterNodeAttributesRequest { + + /** + * Create new instance of GetClusterNodeAttributesRequest. + * + * @return GetClusterNodeAttributesRequest is returned. + */ + public static GetClusterNodeAttributesRequest newInstance() { + return Records.newRecord(GetClusterNodeAttributesRequest.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java new file mode 100644 index 0000000000..cc3cae4c89 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.util.Records; + +import java.util.Set; + +/** + *

+ * The response sent by the ResourceManager to a client requesting + * a node attributes in cluster. + *

+ * + * @see ApplicationClientProtocol#getClusterNodeAttributes + * (GetClusterNodeAttributesRequest) + */ +@Public +@Evolving +public abstract class GetClusterNodeAttributesResponse { + + /** + * Create instance of GetClusterNodeAttributesResponse. + * + * @param attributes + * @return GetClusterNodeAttributesResponse. + */ + public static GetClusterNodeAttributesResponse newInstance( + Set attributes) { + GetClusterNodeAttributesResponse response = + Records.newRecord(GetClusterNodeAttributesResponse.class); + response.setNodeAttributes(attributes); + return response; + } + + /** + * Set node attributes to the response. + * + * @param attributes Node attributes + */ + @Public + @Unstable + public abstract void setNodeAttributes(Set attributes); + + /** + * Get node attributes of the response. + * + * @return Node attributes + */ + @Public + @Unstable + public abstract Set getNodeAttributes(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index edad4d23e2..a82801d620 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3491,7 +3491,7 @@ public static boolean isAclEnabled(Configuration conf) { public static final String FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS = NODE_ATTRIBUTE_PREFIX + "fs-store.impl.class"; /** - * File system not attribute store directory. + * File system node attribute store directory. */ public static final String FS_NODE_ATTRIBUTE_STORE_ROOT_DIR = NODE_ATTRIBUTE_PREFIX + "fs-store.root-dir"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index 81adef1933..eeb884c2c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -64,4 +64,6 @@ service ApplicationClientProtocolService { rpc getResourceProfiles(GetAllResourceProfilesRequestProto) returns (GetAllResourceProfilesResponseProto); rpc getResourceProfile(GetResourceProfileRequestProto) returns (GetResourceProfileResponseProto); rpc getResourceTypeInfo(GetAllResourceTypeInfoRequestProto) returns (GetAllResourceTypeInfoResponseProto); + rpc getClusterNodeAttributes (GetClusterNodeAttributesRequestProto) returns (GetClusterNodeAttributesResponseProto); + rpc getAttributesToNodes (GetAttributesToNodesRequestProto) returns (GetAttributesToNodesResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 815e989cf8..2b796ffe0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -383,6 +383,10 @@ message NodeAttributeProto { optional string attributeValue = 4 [default=""]; } +message AttributeToNodesProto { + required NodeAttributeProto nodeAttribute = 1; + repeated string hostnames = 2; +} enum ContainerTypeProto { APPLICATION_MASTER = 1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index acd452dc79..084457bcf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -260,6 +260,21 @@ message GetClusterNodeLabelsResponseProto { repeated NodeLabelProto nodeLabels = 2; } +message GetClusterNodeAttributesRequestProto { +} + +message GetClusterNodeAttributesResponseProto { + repeated NodeAttributeProto nodeAttributes = 1; +} + +message GetAttributesToNodesRequestProto { + repeated NodeAttributeProto nodeAttributes = 1; +} + +message GetAttributesToNodesResponseProto { + repeated AttributeToNodesProto attributeToNodes = 1; +} + message UpdateApplicationPriorityRequestProto { required ApplicationIdProto applicationId = 1; required PriorityProto applicationPriority = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 26c99e31aa..ca0b7b78f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -33,7 +33,6 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; @@ -52,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -900,4 +900,38 @@ public abstract Resource getResourceProfile(String profile) @Unstable public abstract List getResourceTypeInfo() throws YarnException, IOException; + + /** + *

+ * The interface used by client to get node attributes in the cluster. + *

+ * + * @return cluster node attributes collection + * @throws YarnException when there is a failure in + * {@link ApplicationClientProtocol} + * @throws IOException when there is a failure in + * {@link ApplicationClientProtocol} + */ + @Public + @Unstable + public abstract Set getClusterAttributes() + throws YarnException, IOException; + + /** + *

+ * The interface used by client to get Attributes to nodes mapping + * for specified node attributes in existing cluster. + *

+ * + * @param attributes Attributes for which Attributes to nodes mapping has to + * be retrieved.If empty or null is set then will return + * all attributes to node mapping in cluster. + * @return Attributes to nodes mappings for specific Attributes. + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public abstract Map> getAttributesToNodes( + Set attributes) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 1ceb46209b..2c7496e6a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -22,7 +22,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -52,8 +51,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; @@ -96,6 +97,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -104,7 +106,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; @@ -977,4 +978,20 @@ public List getResourceTypeInfo() GetAllResourceTypeInfoRequest.newInstance(); return rmClient.getResourceTypeInfo(request).getResourceTypeInfo(); } + + @Override + public Set getClusterAttributes() + throws YarnException, IOException { + GetClusterNodeAttributesRequest request = + GetClusterNodeAttributesRequest.newInstance(); + return rmClient.getClusterNodeAttributes(request).getNodeAttributes(); + } + + @Override + public Map> getAttributesToNodes( + Set attributes) throws YarnException, IOException { + GetAttributesToNodesRequest request = + GetAttributesToNodesRequest.newInstance(attributes); + return rmClient.getAttributesToNodes(request).getAttributesToNodes(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index fd5096a7b3..4cf05485a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -43,8 +43,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -107,8 +111,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; @@ -673,4 +681,33 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo( return null; } } + + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + YarnServiceProtos.GetAttributesToNodesRequestProto requestProto = + ((GetAttributesToNodesRequestPBImpl) request).getProto(); + try { + return new GetAttributesToNodesResponsePBImpl( + proxy.getAttributesToNodes(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + YarnServiceProtos.GetClusterNodeAttributesRequestProto requestProto = + ((GetClusterNodeAttributesRequestPBImpl) request).getProto(); + try { + return new GetClusterNodeAttributesResponsePBImpl( + proxy.getClusterNodeAttributes(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index 423287e910..8e53f089ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -35,7 +35,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse; @@ -73,8 +76,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; @@ -184,6 +191,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -694,4 +703,39 @@ public GetAllResourceTypeInfoResponseProto getResourceTypeInfo( throw new ServiceException(ie); } } + + @Override + public GetClusterNodeAttributesResponseProto getClusterNodeAttributes( + RpcController controller, + YarnServiceProtos.GetClusterNodeAttributesRequestProto proto) + throws ServiceException { + GetClusterNodeAttributesRequest req = + new GetClusterNodeAttributesRequestPBImpl(proto); + try { + GetClusterNodeAttributesResponse resp = + real.getClusterNodeAttributes(req); + return ((GetClusterNodeAttributesResponsePBImpl) resp).getProto(); + } catch (YarnException ye) { + throw new ServiceException(ye); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } + + @Override + public GetAttributesToNodesResponseProto getAttributesToNodes( + RpcController controller, + YarnServiceProtos.GetAttributesToNodesRequestProto proto) + throws ServiceException { + GetAttributesToNodesRequestPBImpl req = + new GetAttributesToNodesRequestPBImpl(proto); + try { + GetAttributesToNodesResponse resp = real.getAttributesToNodes(req); + return ((GetAttributesToNodesResponsePBImpl) resp).getProto(); + } catch (YarnException ye) { + throw new ServiceException(ye); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java new file mode 100644 index 0000000000..a84fb44fc9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.protobuf.TextFormat; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesRequestProto; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; + +/** + * Attributes to nodes mapping request. + */ +@Private +@Unstable +public class GetAttributesToNodesRequestPBImpl + extends GetAttributesToNodesRequest { + + private Set nodeAttributes = null; + + private GetAttributesToNodesRequestProto proto = + GetAttributesToNodesRequestProto.getDefaultInstance(); + private GetAttributesToNodesRequestProto.Builder builder = null; + private boolean viaProto = false; + + public GetAttributesToNodesRequestPBImpl() { + builder = GetAttributesToNodesRequestProto.newBuilder(); + } + + public GetAttributesToNodesRequestPBImpl( + GetAttributesToNodesRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetAttributesToNodesRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.nodeAttributes != null) { + addLocalAttributesToProto(); + } + } + + private void addLocalAttributesToProto() { + maybeInitBuilder(); + builder.clearNodeAttributes(); + if (nodeAttributes == null) { + return; + } + Iterable iterable = + () -> new Iterator() { + private Iterator iter = nodeAttributes.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public NodeAttributeProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + builder.addAllNodeAttributes(iterable); + } + + private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) { + return new NodeAttributePBImpl(p); + } + + private NodeAttributeProto convertToProtoFormat(NodeAttribute t) { + return ((NodeAttributePBImpl) t).getProto(); + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetAttributesToNodesRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void initNodeAttributes() { + if (this.nodeAttributes != null) { + return; + } + YarnServiceProtos.GetAttributesToNodesRequestProtoOrBuilder p = + viaProto ? proto : builder; + List nodeAttributesList = p.getNodeAttributesList(); + this.nodeAttributes = new HashSet<>(); + nodeAttributesList + .forEach((v) -> nodeAttributes.add(convertFromProtoFormat(v))); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public void setNodeAttributes(Set attributes) { + maybeInitBuilder(); + if (nodeAttributes == null) { + builder.clearNodeAttributes(); + } + this.nodeAttributes = attributes; + } + + @Override + public Set getNodeAttributes() { + initNodeAttributes(); + return this.nodeAttributes; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java new file mode 100644 index 0000000000..ab6204e9b7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.AttributeToNodesProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; + +/** + * Attributes to nodes response. + */ +@Private +@Unstable +public class GetAttributesToNodesResponsePBImpl + extends GetAttributesToNodesResponse { + + private GetAttributesToNodesResponseProto proto = + GetAttributesToNodesResponseProto.getDefaultInstance(); + private GetAttributesToNodesResponseProto.Builder builder = null; + private boolean viaProto = false; + + private Map> attributesToNodes; + + public GetAttributesToNodesResponsePBImpl() { + this.builder = GetAttributesToNodesResponseProto.newBuilder(); + } + + public GetAttributesToNodesResponsePBImpl( + GetAttributesToNodesResponseProto proto) { + this.proto = proto; + this.viaProto = true; + } + + private void initAttributesToNodes() { + if (this.attributesToNodes != null) { + return; + } + YarnServiceProtos.GetAttributesToNodesResponseProtoOrBuilder p = + viaProto ? proto : builder; + List list = p.getAttributeToNodesList(); + this.attributesToNodes = new HashMap<>(); + + for (AttributeToNodesProto c : list) { + Set setNodes = new HashSet<>(c.getHostnamesList()); + if (!setNodes.isEmpty()) { + this.attributesToNodes + .put(convertFromProtoFormat(c.getNodeAttribute()), setNodes); + } + } + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetAttributesToNodesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void addAttributesToNodesToProto() { + maybeInitBuilder(); + builder.clearAttributeToNodes(); + if (attributesToNodes == null) { + return; + } + Iterable iterable = + () -> new Iterator() { + + private Iterator>> iter = + attributesToNodes.entrySet().iterator(); + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public AttributeToNodesProto next() { + Map.Entry> now = iter.next(); + Set hostNames = new HashSet<>(); + for (String host : now.getValue()) { + hostNames.add(host); + } + return AttributeToNodesProto.newBuilder() + .setNodeAttribute(convertToProtoFormat(now.getKey())) + .addAllHostnames(hostNames).build(); + } + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + }; + builder.addAllAttributeToNodes(iterable); + } + + private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) { + return new NodeAttributePBImpl(p); + } + + private NodeAttributeProto convertToProtoFormat(NodeAttribute t) { + return ((NodeAttributePBImpl) t).getProto(); + } + + private void mergeLocalToBuilder() { + if (this.attributesToNodes != null) { + addAttributesToNodesToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + public GetAttributesToNodesResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + assert false : "hashCode not designed"; + return 0; + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public void setAttributeToNodes(Map> map) { + initAttributesToNodes(); + attributesToNodes.clear(); + attributesToNodes.putAll(map); + } + + @Override + public Map> getAttributesToNodes() { + initAttributesToNodes(); + return this.attributesToNodes; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesRequestPBImpl.java new file mode 100644 index 0000000000..bf5ab4084e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesRequestPBImpl.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.protobuf.TextFormat; +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesRequestProto; + +/** + * Request to get cluster node attributes. + */ +@Private +@Unstable +public class GetClusterNodeAttributesRequestPBImpl + extends GetClusterNodeAttributesRequest { + + private GetClusterNodeAttributesRequestProto proto = + GetClusterNodeAttributesRequestProto.getDefaultInstance(); + private GetClusterNodeAttributesRequestProto.Builder builder = null; + private boolean viaProto = false; + + public GetClusterNodeAttributesRequestPBImpl() { + builder = GetClusterNodeAttributesRequestProto.newBuilder(); + } + + public GetClusterNodeAttributesRequestPBImpl( + GetClusterNodeAttributesRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetClusterNodeAttributesRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java new file mode 100644 index 0000000000..385155f3f2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Cluster node attributes response. + */ +@Private +@Unstable +public class GetClusterNodeAttributesResponsePBImpl + extends GetClusterNodeAttributesResponse { + + private GetClusterNodeAttributesResponseProto proto = + GetClusterNodeAttributesResponseProto.getDefaultInstance(); + private GetClusterNodeAttributesResponseProto.Builder builder = null; + private Set updatedNodeAttributes; + private boolean viaProto = false; + + public GetClusterNodeAttributesResponsePBImpl() { + builder = GetClusterNodeAttributesResponseProto.newBuilder(); + } + + public GetClusterNodeAttributesResponsePBImpl( + GetClusterNodeAttributesResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized GetClusterNodeAttributesResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.updatedNodeAttributes != null) { + addNodeAttributesToProto(); + } + } + + private void addNodeAttributesToProto() { + maybeInitBuilder(); + builder.clearNodeAttributes(); + List protoList = new ArrayList<>(); + for (NodeAttribute r : this.updatedNodeAttributes) { + protoList.add(convertToProtoFormat(r)); + } + builder.addAllNodeAttributes(protoList); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public int hashCode() { + assert false : "hashCode not designed"; + return 0; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetClusterNodeAttributesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public synchronized void setNodeAttributes(Set attributes) { + maybeInitBuilder(); + this.updatedNodeAttributes = new HashSet<>(); + if (attributes == null) { + builder.clearNodeAttributes(); + return; + } + this.updatedNodeAttributes.addAll(attributes); + } + + @Override + public synchronized Set getNodeAttributes() { + if (this.updatedNodeAttributes != null) { + return this.updatedNodeAttributes; + } + initLocalNodeAttributes(); + return this.updatedNodeAttributes; + } + + private void initLocalNodeAttributes() { + YarnServiceProtos.GetClusterNodeAttributesResponseProtoOrBuilder p = + viaProto ? proto : builder; + List attributesProtoList = p.getNodeAttributesList(); + this.updatedNodeAttributes = new HashSet<>(); + for (NodeAttributeProto r : attributesProtoList) { + this.updatedNodeAttributes.add(convertFromProtoFormat(r)); + } + } + + private NodeAttribute convertFromProtoFormat(NodeAttributeProto p) { + return new NodeAttributePBImpl(p); + } + + private NodeAttributeProto convertToProtoFormat(NodeAttribute t) { + return ((NodeAttributePBImpl) t).getProto(); + } + + @Override + public String toString() { + return getProto().toString(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java index ec7d30d4f4..3816051ca9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java @@ -88,12 +88,13 @@ public abstract Set getClusterNodeAttributes( /** * Given a attribute set, return what all Nodes have attribute mapped to it. + * If the attributes set is null or empty, all attributes mapping are + * returned. * - * @return a Map, of attribute to set of hostnames + * @return a Map of attributes to set of hostnames. */ - //TODO need to handle as part of REST patch. -/* public abstract Map> getAttributesToNodes( - Set attributes);*/ + public abstract Map> getAttributesToNodes( + Set attributes); /** * NodeAttribute to AttributeValue Map. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 1af4191cc6..9397dd83f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -53,8 +53,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; @@ -1259,4 +1263,28 @@ public void testNodesToAttributesMappingRequestPBImpl() throws Exception { validatePBImplRecord(NodesToAttributesMappingRequestPBImpl.class, NodesToAttributesMappingRequestProto.class); } + + @Test + public void testGetAttributesToNodesRequestPBImpl() throws Exception { + validatePBImplRecord(GetAttributesToNodesRequestPBImpl.class, + YarnServiceProtos.GetAttributesToNodesRequestProto.class); + } + + @Test + public void testGetAttributesToNodesResponsePBImpl() throws Exception { + validatePBImplRecord(GetAttributesToNodesResponsePBImpl.class, + YarnServiceProtos.GetAttributesToNodesResponseProto.class); + } + + @Test + public void testGetClusterNodeAttributesRequestPBImpl() throws Exception { + validatePBImplRecord(GetClusterNodeAttributesRequestPBImpl.class, + YarnServiceProtos.GetClusterNodeAttributesRequestProto.class); + } + + @Test + public void testGetClusterNodeAttributesResponsePBImpl() throws Exception { + validatePBImplRecord(GetClusterNodeAttributesResponsePBImpl.class, + YarnServiceProtos.GetClusterNodeAttributesResponseProto.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index b9cebfe066..47b51f8cab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -53,8 +53,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -895,6 +899,19 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo( return null; } + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + return null; + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + return null; + } + @Override public NodesToAttributesMappingResponse mapAttributesToNodes(NodesToAttributesMappingRequest request) throws YarnException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index be997534dd..05c11cf496 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -66,8 +66,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -127,6 +131,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; @@ -148,6 +153,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys; @@ -1838,6 +1844,30 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo( return response; } + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + NodeAttributesManager attributesManager = + rmContext.getNodeAttributesManager(); + GetAttributesToNodesResponse response = GetAttributesToNodesResponse + .newInstance(attributesManager + .getAttributesToNodes(request.getNodeAttributes())); + return response; + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + NodeAttributesManager attributesManager = + rmContext.getNodeAttributesManager(); + Set attributes = + attributesManager.getClusterNodeAttributes(null); + GetClusterNodeAttributesResponse response = + GetClusterNodeAttributesResponse.newInstance(attributes); + return response; + } + @VisibleForTesting public void setDisplayPerUserApps(boolean displayPerUserApps) { this.filterAppsByUser = displayPerUserApps; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java index b4686e638a..09671f16b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java @@ -36,7 +36,7 @@ import java.util.List; import com.google.common.base.Strings; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -367,19 +367,26 @@ public Set getClusterNodeAttributes(Set prefix) { return attributes; } - // TODO need to handle as part of REST patch. - /* - * @Override public Map> getAttributesToNodes( - * Set attributes) { try { readLock.lock(); boolean - * fetchAllAttributes = (attributes == null || attributes.isEmpty()); - * Map> attributesToNodes = new HashMap<>(); for - * (Entry attributeEntry : - * attributeCollections .entrySet()) { if (fetchAllAttributes || - * attributes.contains(attributeEntry.getKey())) { - * attributesToNodes.put(attributeEntry.getKey(), - * attributeEntry.getValue().getAssociatedNodeIds()); } } return - * attributesToNodes; } finally { readLock.unlock(); } } - */ + @Override + public Map> getAttributesToNodes( + Set attributes) { + try { + readLock.lock(); + boolean fetchAllAttributes = (attributes == null || attributes.isEmpty()); + Map> attributesToNodes = new HashMap<>(); + for (Entry attributeEntry : + clusterAttributes.entrySet()) { + if (fetchAllAttributes || attributes + .contains(attributeEntry.getKey())) { + attributesToNodes.put(attributeEntry.getKey(), + attributeEntry.getValue().getAssociatedNodeIds()); + } + } + return attributesToNodes; + } finally { + readLock.unlock(); + } + } public Resource getResourceByAttribute(NodeAttribute attribute) { try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 6644e44453..4a6b366e8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -18,6 +18,13 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -1999,6 +2006,125 @@ protected ClientRMService createClientRMService() { rm.close(); } + @Test(timeout = 120000) + public void testGetClusterNodeAttributes() throws IOException, YarnException { + MockRM rm = new MockRM() { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, + this.applicationACLsManager, this.queueACLsManager, + this.getRMContext().getRMDelegationTokenSecretManager()); + } + }; + rm.start(); + + NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager(); + NodeId host1 = NodeId.newInstance("host1", 0); + NodeId host2 = NodeId.newInstance("host2", 0); + NodeAttribute gpu = NodeAttribute + .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvida"); + NodeAttribute os = NodeAttribute + .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", + NodeAttributeType.STRING, "windows64"); + NodeAttribute docker = NodeAttribute + .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", + NodeAttributeType.STRING, "docker0"); + Map> nodes = new HashMap<>(); + nodes.put(host1.getHost(), ImmutableSet.of(gpu, os)); + nodes.put(host2.getHost(), ImmutableSet.of(docker)); + mgr.addNodeAttributes(nodes); + // Create a client. + Configuration conf = new Configuration(); + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); + LOG.info("Connecting to ResourceManager at " + rmAddress); + ApplicationClientProtocol client = (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); + + GetClusterNodeAttributesRequest request = + GetClusterNodeAttributesRequest.newInstance(); + GetClusterNodeAttributesResponse response = + client.getClusterNodeAttributes(request); + Set attributes = response.getNodeAttributes(); + Assert.assertEquals("Size not correct", 3, attributes.size()); + Assert.assertTrue(attributes.contains(gpu)); + Assert.assertTrue(attributes.contains(os)); + Assert.assertTrue(attributes.contains(docker)); + rpc.stopProxy(client, conf); + rm.close(); + } + + @Test(timeout = 120000) + public void testGetAttributesToNodes() throws IOException, YarnException { + MockRM rm = new MockRM() { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, + this.applicationACLsManager, this.queueACLsManager, + this.getRMContext().getRMDelegationTokenSecretManager()); + } + }; + rm.start(); + + NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager(); + String node1 = "host1"; + String node2 = "host2"; + NodeAttribute gpu = NodeAttribute + .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvida"); + NodeAttribute os = NodeAttribute + .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", + NodeAttributeType.STRING, "windows64"); + NodeAttribute docker = NodeAttribute + .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", + NodeAttributeType.STRING, "docker0"); + NodeAttribute dist = NodeAttribute + .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION", + NodeAttributeType.STRING, "3_0_2"); + Map> nodes = new HashMap<>(); + nodes.put(node1, ImmutableSet.of(gpu, os, dist)); + nodes.put(node2, ImmutableSet.of(docker, dist)); + mgr.addNodeAttributes(nodes); + // Create a client. + Configuration conf = new Configuration(); + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); + LOG.info("Connecting to ResourceManager at " + rmAddress); + ApplicationClientProtocol client = (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); + + GetAttributesToNodesRequest request = + GetAttributesToNodesRequest.newInstance(); + GetAttributesToNodesResponse response = + client.getAttributesToNodes(request); + Map> attrs = response.getAttributesToNodes(); + Assert.assertEquals(response.getAttributesToNodes().size(), 4); + Assert.assertEquals(attrs.get(dist).size(), 2); + Assert.assertEquals(attrs.get(os).size(), 1); + Assert.assertEquals(attrs.get(gpu).size(), 1); + Assert.assertTrue(attrs.get(dist).contains(node1)); + Assert.assertTrue(attrs.get(dist).contains(node2)); + Assert.assertTrue(attrs.get(docker).contains(node2)); + + GetAttributesToNodesRequest request2 = + GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker)); + GetAttributesToNodesResponse response2 = + client.getAttributesToNodes(request2); + Map> attrs2 = response2.getAttributesToNodes(); + Assert.assertEquals(response2.getAttributesToNodes().size(), 1); + Assert.assertTrue(attrs.get(docker).contains(node2)); + + GetAttributesToNodesRequest request3 = + GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker, os)); + GetAttributesToNodesResponse response3 = + client.getAttributesToNodes(request3); + Map> attrs3 = response3.getAttributesToNodes(); + Assert.assertEquals(response3.getAttributesToNodes().size(), 2); + Assert.assertTrue(attrs.get(os).contains(node1)); + Assert.assertTrue(attrs.get(docker).contains(node2)); + rpc.stopProxy(client, conf); + rm.close(); + } + @Test(timeout = 120000) public void testUpdatePriorityAndKillAppWithZeroClusterResource() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java index 5ce4803f60..f6adb434d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java @@ -39,8 +39,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -327,6 +331,19 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo( return clientRMProxy.getResourceTypeInfo(request); } + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + return clientRMProxy.getAttributesToNodes(request); + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + return clientRMProxy.getClusterNodeAttributes(request); + } + @VisibleForTesting public void setRMClient(ApplicationClientProtocol clientRM) { this.clientRMProxy = clientRM; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 8b48c8c5b2..bf006a4e68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -55,8 +55,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -826,4 +830,17 @@ public void shutdown() { executorService.shutdown(); super.shutdown(); } + + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index bbb8047d98..3237dd4875 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -50,8 +50,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -430,6 +434,21 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo( return pipeline.getRootInterceptor().getResourceTypeInfo(request); } + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + RequestInterceptorChainWrapper pipeline = getInterceptorChain(); + return pipeline.getRootInterceptor().getAttributesToNodes(request); + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + RequestInterceptorChainWrapper pipeline = getInterceptorChain(); + return pipeline.getRootInterceptor().getClusterNodeAttributes(request); + } + @VisibleForTesting protected RequestInterceptorChainWrapper getInterceptorChain() throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java index cb1b529d8e..96da4c430b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java @@ -36,8 +36,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -288,4 +292,17 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo( GetAllResourceTypeInfoRequest request) throws YarnException, IOException { return getNextInterceptor().getResourceTypeInfo(request); } + + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + return getNextInterceptor().getAttributesToNodes(request); + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + return getNextInterceptor().getClusterNodeAttributes(request); + } }