YARN-8100. Support API interface to query cluster attributes and attribute to nodes. Contributed by Bibin A Chundatt.
This commit is contained in:
parent
b9890d1f66
commit
0a01b1350d
@ -58,6 +58,7 @@
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||
import org.apache.hadoop.yarn.api.records.NodeLabel;
|
||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||
@ -538,4 +539,16 @@ public List<ResourceTypeInfo> getResourceTypeInfo()
|
||||
throws YarnException, IOException {
|
||||
return client.getResourceTypeInfo();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<NodeAttribute> getClusterAttributes()
|
||||
throws YarnException, IOException {
|
||||
return client.getClusterAttributes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||
Set<NodeAttribute> attributes) throws YarnException, IOException {
|
||||
return client.getAttributesToNodes(attributes);
|
||||
}
|
||||
}
|
||||
|
@ -82,8 +82,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
@ -521,6 +525,19 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
||||
throws YarnException, IOException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||
GetClusterNodeAttributesRequest request)
|
||||
throws YarnException, IOException {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
class HistoryService extends AMService implements HSClientProtocol {
|
||||
|
@ -27,8 +27,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
@ -642,4 +646,37 @@ GetResourceProfileResponse getResourceProfile(
|
||||
@Unstable
|
||||
GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
||||
GetAllResourceTypeInfoRequest request) throws YarnException, IOException;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* The interface used by client to get attributes to nodes mappings
|
||||
* available in ResourceManager.
|
||||
* </p>
|
||||
*
|
||||
* @param request request to get details of attributes to nodes mapping.
|
||||
* @return Response containing the details of attributes to nodes mappings.
|
||||
* @throws YarnException if any error happens inside YARN
|
||||
* @throws IOException incase of other errors
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
GetAttributesToNodesResponse getAttributesToNodes(
|
||||
GetAttributesToNodesRequest request) throws YarnException, IOException;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* The interface used by client to get node attributes available in
|
||||
* ResourceManager.
|
||||
* </p>
|
||||
*
|
||||
* @param request request to get node attributes collection of this cluster.
|
||||
* @return Response containing node attributes collection.
|
||||
* @throws YarnException if any error happens inside YARN.
|
||||
* @throws IOException incase of other errors.
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||
GetClusterNodeAttributesRequest request)
|
||||
throws YarnException, IOException;
|
||||
}
|
||||
|
@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.api.protocolrecords;
|
||||
|
||||
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* The request from clients to get attribtues to nodes mapping
|
||||
* in the cluster from the <code>ResourceManager</code>.
|
||||
* </p>
|
||||
*
|
||||
* @see ApplicationClientProtocol#getAttributesToNodes
|
||||
* (GetAttributesToNodesRequest)
|
||||
*/
|
||||
@Public
|
||||
@Evolving
|
||||
public abstract class GetAttributesToNodesRequest {
|
||||
|
||||
public static GetAttributesToNodesRequest newInstance() {
|
||||
return Records.newRecord(GetAttributesToNodesRequest.class);
|
||||
}
|
||||
|
||||
public static GetAttributesToNodesRequest newInstance(
|
||||
Set<NodeAttribute> attributes) {
|
||||
GetAttributesToNodesRequest request =
|
||||
Records.newRecord(GetAttributesToNodesRequest.class);
|
||||
request.setNodeAttributes(attributes);
|
||||
return request;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set node attributes for which the mapping is required.
|
||||
*
|
||||
* @param attributes Set<NodeAttribute> provided.
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public abstract void setNodeAttributes(Set<NodeAttribute> attributes);
|
||||
|
||||
/**
|
||||
* Get node attributes for which mapping mapping is required.
|
||||
*
|
||||
* @return Set<NodeAttribute>
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public abstract Set<NodeAttribute> getNodeAttributes();
|
||||
}
|
@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.api.protocolrecords;
|
||||
|
||||
import static org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||
import static org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* The response sent by the <code>ResourceManager</code> to a client requesting
|
||||
* attributes to hostname mapping.
|
||||
* </p>
|
||||
*
|
||||
* @see ApplicationClientProtocol#getAttributesToNodes
|
||||
* (GetAttributesToNodesRequest)
|
||||
*/
|
||||
@Public
|
||||
@Evolving
|
||||
public abstract class GetAttributesToNodesResponse {
|
||||
public static GetAttributesToNodesResponse newInstance(
|
||||
Map<NodeAttribute, Set<String>> map) {
|
||||
GetAttributesToNodesResponse response =
|
||||
Records.newRecord(GetAttributesToNodesResponse.class);
|
||||
response.setAttributeToNodes(map);
|
||||
return response;
|
||||
}
|
||||
|
||||
@Public
|
||||
@Evolving
|
||||
public abstract void setAttributeToNodes(Map<NodeAttribute, Set<String>> map);
|
||||
|
||||
/*
|
||||
* Get attributes to node hostname mapping.
|
||||
*
|
||||
* @return Map<NodeAttribute, Set<String>> node attributes to hostname
|
||||
* mapping.
|
||||
*/
|
||||
@Public
|
||||
@Evolving
|
||||
public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes();
|
||||
}
|
@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.api.protocolrecords;
|
||||
|
||||
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* The request from clients to get node attributes in the cluster from the
|
||||
* <code>ResourceManager</code>.
|
||||
* </p>
|
||||
*
|
||||
* @see ApplicationClientProtocol#getClusterNodeAttributes
|
||||
* (GetClusterNodeAttributesRequest)
|
||||
*/
|
||||
@Public
|
||||
@Evolving
|
||||
public abstract class GetClusterNodeAttributesRequest {
|
||||
|
||||
/**
|
||||
* Create new instance of GetClusterNodeAttributesRequest.
|
||||
*
|
||||
* @return GetClusterNodeAttributesRequest is returned.
|
||||
*/
|
||||
public static GetClusterNodeAttributesRequest newInstance() {
|
||||
return Records.newRecord(GetClusterNodeAttributesRequest.class);
|
||||
}
|
||||
}
|
@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.api.protocolrecords;
|
||||
|
||||
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* The response sent by the <code>ResourceManager</code> to a client requesting
|
||||
* a node attributes in cluster.
|
||||
* </p>
|
||||
*
|
||||
* @see ApplicationClientProtocol#getClusterNodeAttributes
|
||||
* (GetClusterNodeAttributesRequest)
|
||||
*/
|
||||
@Public
|
||||
@Evolving
|
||||
public abstract class GetClusterNodeAttributesResponse {
|
||||
|
||||
/**
|
||||
* Create instance of GetClusterNodeAttributesResponse.
|
||||
*
|
||||
* @param attributes
|
||||
* @return GetClusterNodeAttributesResponse.
|
||||
*/
|
||||
public static GetClusterNodeAttributesResponse newInstance(
|
||||
Set<NodeAttribute> attributes) {
|
||||
GetClusterNodeAttributesResponse response =
|
||||
Records.newRecord(GetClusterNodeAttributesResponse.class);
|
||||
response.setNodeAttributes(attributes);
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set node attributes to the response.
|
||||
*
|
||||
* @param attributes Node attributes
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public abstract void setNodeAttributes(Set<NodeAttribute> attributes);
|
||||
|
||||
/**
|
||||
* Get node attributes of the response.
|
||||
*
|
||||
* @return Node attributes
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public abstract Set<NodeAttribute> getNodeAttributes();
|
||||
}
|
@ -3491,7 +3491,7 @@ public static boolean isAclEnabled(Configuration conf) {
|
||||
public static final String FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS =
|
||||
NODE_ATTRIBUTE_PREFIX + "fs-store.impl.class";
|
||||
/**
|
||||
* File system not attribute store directory.
|
||||
* File system node attribute store directory.
|
||||
*/
|
||||
public static final String FS_NODE_ATTRIBUTE_STORE_ROOT_DIR =
|
||||
NODE_ATTRIBUTE_PREFIX + "fs-store.root-dir";
|
||||
|
@ -64,4 +64,6 @@ service ApplicationClientProtocolService {
|
||||
rpc getResourceProfiles(GetAllResourceProfilesRequestProto) returns (GetAllResourceProfilesResponseProto);
|
||||
rpc getResourceProfile(GetResourceProfileRequestProto) returns (GetResourceProfileResponseProto);
|
||||
rpc getResourceTypeInfo(GetAllResourceTypeInfoRequestProto) returns (GetAllResourceTypeInfoResponseProto);
|
||||
rpc getClusterNodeAttributes (GetClusterNodeAttributesRequestProto) returns (GetClusterNodeAttributesResponseProto);
|
||||
rpc getAttributesToNodes (GetAttributesToNodesRequestProto) returns (GetAttributesToNodesResponseProto);
|
||||
}
|
||||
|
@ -383,6 +383,10 @@ message NodeAttributeProto {
|
||||
optional string attributeValue = 4 [default=""];
|
||||
}
|
||||
|
||||
message AttributeToNodesProto {
|
||||
required NodeAttributeProto nodeAttribute = 1;
|
||||
repeated string hostnames = 2;
|
||||
}
|
||||
|
||||
enum ContainerTypeProto {
|
||||
APPLICATION_MASTER = 1;
|
||||
|
@ -260,6 +260,21 @@ message GetClusterNodeLabelsResponseProto {
|
||||
repeated NodeLabelProto nodeLabels = 2;
|
||||
}
|
||||
|
||||
message GetClusterNodeAttributesRequestProto {
|
||||
}
|
||||
|
||||
message GetClusterNodeAttributesResponseProto {
|
||||
repeated NodeAttributeProto nodeAttributes = 1;
|
||||
}
|
||||
|
||||
message GetAttributesToNodesRequestProto {
|
||||
repeated NodeAttributeProto nodeAttributes = 1;
|
||||
}
|
||||
|
||||
message GetAttributesToNodesResponseProto {
|
||||
repeated AttributeToNodesProto attributeToNodes = 1;
|
||||
}
|
||||
|
||||
message UpdateApplicationPriorityRequestProto {
|
||||
required ApplicationIdProto applicationId = 1;
|
||||
required PriorityProto applicationPriority = 2;
|
||||
|
@ -33,7 +33,6 @@
|
||||
import org.apache.hadoop.service.AbstractService;
|
||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
|
||||
@ -52,6 +51,7 @@
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||
import org.apache.hadoop.yarn.api.records.NodeLabel;
|
||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||
@ -900,4 +900,38 @@ public abstract Resource getResourceProfile(String profile)
|
||||
@Unstable
|
||||
public abstract List<ResourceTypeInfo> getResourceTypeInfo()
|
||||
throws YarnException, IOException;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* The interface used by client to get node attributes in the cluster.
|
||||
* </p>
|
||||
*
|
||||
* @return cluster node attributes collection
|
||||
* @throws YarnException when there is a failure in
|
||||
* {@link ApplicationClientProtocol}
|
||||
* @throws IOException when there is a failure in
|
||||
* {@link ApplicationClientProtocol}
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public abstract Set<NodeAttribute> getClusterAttributes()
|
||||
throws YarnException, IOException;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* The interface used by client to get Attributes to nodes mapping
|
||||
* for specified node attributes in existing cluster.
|
||||
* </p>
|
||||
*
|
||||
* @param attributes Attributes for which Attributes to nodes mapping has to
|
||||
* be retrieved.If empty or null is set then will return
|
||||
* all attributes to node mapping in cluster.
|
||||
* @return Attributes to nodes mappings for specific Attributes.
|
||||
* @throws YarnException
|
||||
* @throws IOException
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||
Set<NodeAttribute> attributes) throws YarnException, IOException;
|
||||
}
|
||||
|
@ -22,7 +22,6 @@
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
@ -52,8 +51,10 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
|
||||
@ -96,6 +97,7 @@
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||
import org.apache.hadoop.yarn.api.records.NodeLabel;
|
||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||
@ -104,7 +106,6 @@
|
||||
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
|
||||
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
|
||||
import org.apache.hadoop.yarn.api.records.Token;
|
||||
@ -977,4 +978,20 @@ public List<ResourceTypeInfo> getResourceTypeInfo()
|
||||
GetAllResourceTypeInfoRequest.newInstance();
|
||||
return rmClient.getResourceTypeInfo(request).getResourceTypeInfo();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<NodeAttribute> getClusterAttributes()
|
||||
throws YarnException, IOException {
|
||||
GetClusterNodeAttributesRequest request =
|
||||
GetClusterNodeAttributesRequest.newInstance();
|
||||
return rmClient.getClusterNodeAttributes(request).getNodeAttributes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||
Set<NodeAttribute> attributes) throws YarnException, IOException {
|
||||
GetAttributesToNodesRequest request =
|
||||
GetAttributesToNodesRequest.newInstance(attributes);
|
||||
return rmClient.getAttributesToNodes(request).getAttributesToNodes();
|
||||
}
|
||||
}
|
||||
|
@ -43,8 +43,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
@ -107,8 +111,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl;
|
||||
@ -673,4 +681,33 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||
YarnServiceProtos.GetAttributesToNodesRequestProto requestProto =
|
||||
((GetAttributesToNodesRequestPBImpl) request).getProto();
|
||||
try {
|
||||
return new GetAttributesToNodesResponsePBImpl(
|
||||
proxy.getAttributesToNodes(null, requestProto));
|
||||
} catch (ServiceException e) {
|
||||
RPCUtil.unwrapAndThrowException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||
GetClusterNodeAttributesRequest request)
|
||||
throws YarnException, IOException {
|
||||
YarnServiceProtos.GetClusterNodeAttributesRequestProto requestProto =
|
||||
((GetClusterNodeAttributesRequestPBImpl) request).getProto();
|
||||
try {
|
||||
return new GetClusterNodeAttributesResponsePBImpl(
|
||||
proxy.getClusterNodeAttributes(null, requestProto));
|
||||
} catch (ServiceException e) {
|
||||
RPCUtil.unwrapAndThrowException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -35,7 +35,10 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse;
|
||||
@ -73,8 +76,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl;
|
||||
@ -184,6 +191,8 @@
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesRequestProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto;
|
||||
|
||||
import com.google.protobuf.RpcController;
|
||||
import com.google.protobuf.ServiceException;
|
||||
@ -694,4 +703,39 @@ public GetAllResourceTypeInfoResponseProto getResourceTypeInfo(
|
||||
throw new ServiceException(ie);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetClusterNodeAttributesResponseProto getClusterNodeAttributes(
|
||||
RpcController controller,
|
||||
YarnServiceProtos.GetClusterNodeAttributesRequestProto proto)
|
||||
throws ServiceException {
|
||||
GetClusterNodeAttributesRequest req =
|
||||
new GetClusterNodeAttributesRequestPBImpl(proto);
|
||||
try {
|
||||
GetClusterNodeAttributesResponse resp =
|
||||
real.getClusterNodeAttributes(req);
|
||||
return ((GetClusterNodeAttributesResponsePBImpl) resp).getProto();
|
||||
} catch (YarnException ye) {
|
||||
throw new ServiceException(ye);
|
||||
} catch (IOException ie) {
|
||||
throw new ServiceException(ie);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetAttributesToNodesResponseProto getAttributesToNodes(
|
||||
RpcController controller,
|
||||
YarnServiceProtos.GetAttributesToNodesRequestProto proto)
|
||||
throws ServiceException {
|
||||
GetAttributesToNodesRequestPBImpl req =
|
||||
new GetAttributesToNodesRequestPBImpl(proto);
|
||||
try {
|
||||
GetAttributesToNodesResponse resp = real.getAttributesToNodes(req);
|
||||
return ((GetAttributesToNodesResponsePBImpl) resp).getProto();
|
||||
} catch (YarnException ye) {
|
||||
throw new ServiceException(ye);
|
||||
} catch (IOException ie) {
|
||||
throw new ServiceException(ie);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,175 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||
|
||||
import com.google.protobuf.TextFormat;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesRequestProto;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||
|
||||
/**
|
||||
* Attributes to nodes mapping request.
|
||||
*/
|
||||
@Private
|
||||
@Unstable
|
||||
public class GetAttributesToNodesRequestPBImpl
|
||||
extends GetAttributesToNodesRequest {
|
||||
|
||||
private Set<NodeAttribute> nodeAttributes = null;
|
||||
|
||||
private GetAttributesToNodesRequestProto proto =
|
||||
GetAttributesToNodesRequestProto.getDefaultInstance();
|
||||
private GetAttributesToNodesRequestProto.Builder builder = null;
|
||||
private boolean viaProto = false;
|
||||
|
||||
public GetAttributesToNodesRequestPBImpl() {
|
||||
builder = GetAttributesToNodesRequestProto.newBuilder();
|
||||
}
|
||||
|
||||
public GetAttributesToNodesRequestPBImpl(
|
||||
GetAttributesToNodesRequestProto proto) {
|
||||
this.proto = proto;
|
||||
viaProto = true;
|
||||
}
|
||||
|
||||
public GetAttributesToNodesRequestProto getProto() {
|
||||
mergeLocalToProto();
|
||||
proto = viaProto ? proto : builder.build();
|
||||
viaProto = true;
|
||||
return proto;
|
||||
}
|
||||
|
||||
private void mergeLocalToProto() {
|
||||
if (viaProto) {
|
||||
maybeInitBuilder();
|
||||
}
|
||||
mergeLocalToBuilder();
|
||||
proto = builder.build();
|
||||
viaProto = true;
|
||||
}
|
||||
|
||||
private void mergeLocalToBuilder() {
|
||||
if (this.nodeAttributes != null) {
|
||||
addLocalAttributesToProto();
|
||||
}
|
||||
}
|
||||
|
||||
private void addLocalAttributesToProto() {
|
||||
maybeInitBuilder();
|
||||
builder.clearNodeAttributes();
|
||||
if (nodeAttributes == null) {
|
||||
return;
|
||||
}
|
||||
Iterable<NodeAttributeProto> iterable =
|
||||
() -> new Iterator<NodeAttributeProto>() {
|
||||
private Iterator<NodeAttribute> iter = nodeAttributes.iterator();
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return iter.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public NodeAttributeProto next() {
|
||||
return convertToProtoFormat(iter.next());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
builder.addAllNodeAttributes(iterable);
|
||||
}
|
||||
|
||||
private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) {
|
||||
return new NodeAttributePBImpl(p);
|
||||
}
|
||||
|
||||
private NodeAttributeProto convertToProtoFormat(NodeAttribute t) {
|
||||
return ((NodeAttributePBImpl) t).getProto();
|
||||
}
|
||||
|
||||
private void maybeInitBuilder() {
|
||||
if (viaProto || builder == null) {
|
||||
builder = GetAttributesToNodesRequestProto.newBuilder(proto);
|
||||
}
|
||||
viaProto = false;
|
||||
}
|
||||
|
||||
private void initNodeAttributes() {
|
||||
if (this.nodeAttributes != null) {
|
||||
return;
|
||||
}
|
||||
YarnServiceProtos.GetAttributesToNodesRequestProtoOrBuilder p =
|
||||
viaProto ? proto : builder;
|
||||
List<NodeAttributeProto> nodeAttributesList = p.getNodeAttributesList();
|
||||
this.nodeAttributes = new HashSet<>();
|
||||
nodeAttributesList
|
||||
.forEach((v) -> nodeAttributes.add(convertFromProtoFormat(v)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return getProto().hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
if (other == null) {
|
||||
return false;
|
||||
}
|
||||
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return TextFormat.shortDebugString(getProto());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setNodeAttributes(Set<NodeAttribute> attributes) {
|
||||
maybeInitBuilder();
|
||||
if (nodeAttributes == null) {
|
||||
builder.clearNodeAttributes();
|
||||
}
|
||||
this.nodeAttributes = attributes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<NodeAttribute> getNodeAttributes() {
|
||||
initNodeAttributes();
|
||||
return this.nodeAttributes;
|
||||
}
|
||||
}
|
@ -0,0 +1,184 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.AttributeToNodesProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||
|
||||
/**
|
||||
* Attributes to nodes response.
|
||||
*/
|
||||
@Private
|
||||
@Unstable
|
||||
public class GetAttributesToNodesResponsePBImpl
|
||||
extends GetAttributesToNodesResponse {
|
||||
|
||||
private GetAttributesToNodesResponseProto proto =
|
||||
GetAttributesToNodesResponseProto.getDefaultInstance();
|
||||
private GetAttributesToNodesResponseProto.Builder builder = null;
|
||||
private boolean viaProto = false;
|
||||
|
||||
private Map<NodeAttribute, Set<String>> attributesToNodes;
|
||||
|
||||
public GetAttributesToNodesResponsePBImpl() {
|
||||
this.builder = GetAttributesToNodesResponseProto.newBuilder();
|
||||
}
|
||||
|
||||
public GetAttributesToNodesResponsePBImpl(
|
||||
GetAttributesToNodesResponseProto proto) {
|
||||
this.proto = proto;
|
||||
this.viaProto = true;
|
||||
}
|
||||
|
||||
private void initAttributesToNodes() {
|
||||
if (this.attributesToNodes != null) {
|
||||
return;
|
||||
}
|
||||
YarnServiceProtos.GetAttributesToNodesResponseProtoOrBuilder p =
|
||||
viaProto ? proto : builder;
|
||||
List<AttributeToNodesProto> list = p.getAttributeToNodesList();
|
||||
this.attributesToNodes = new HashMap<>();
|
||||
|
||||
for (AttributeToNodesProto c : list) {
|
||||
Set<String> setNodes = new HashSet<>(c.getHostnamesList());
|
||||
if (!setNodes.isEmpty()) {
|
||||
this.attributesToNodes
|
||||
.put(convertFromProtoFormat(c.getNodeAttribute()), setNodes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void maybeInitBuilder() {
|
||||
if (viaProto || builder == null) {
|
||||
builder = GetAttributesToNodesResponseProto.newBuilder(proto);
|
||||
}
|
||||
viaProto = false;
|
||||
}
|
||||
|
||||
private void addAttributesToNodesToProto() {
|
||||
maybeInitBuilder();
|
||||
builder.clearAttributeToNodes();
|
||||
if (attributesToNodes == null) {
|
||||
return;
|
||||
}
|
||||
Iterable<AttributeToNodesProto> iterable =
|
||||
() -> new Iterator<AttributeToNodesProto>() {
|
||||
|
||||
private Iterator<Map.Entry<NodeAttribute, Set<String>>> iter =
|
||||
attributesToNodes.entrySet().iterator();
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AttributeToNodesProto next() {
|
||||
Map.Entry<NodeAttribute, Set<String>> now = iter.next();
|
||||
Set<String> hostNames = new HashSet<>();
|
||||
for (String host : now.getValue()) {
|
||||
hostNames.add(host);
|
||||
}
|
||||
return AttributeToNodesProto.newBuilder()
|
||||
.setNodeAttribute(convertToProtoFormat(now.getKey()))
|
||||
.addAllHostnames(hostNames).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return iter.hasNext();
|
||||
}
|
||||
};
|
||||
builder.addAllAttributeToNodes(iterable);
|
||||
}
|
||||
|
||||
private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) {
|
||||
return new NodeAttributePBImpl(p);
|
||||
}
|
||||
|
||||
private NodeAttributeProto convertToProtoFormat(NodeAttribute t) {
|
||||
return ((NodeAttributePBImpl) t).getProto();
|
||||
}
|
||||
|
||||
private void mergeLocalToBuilder() {
|
||||
if (this.attributesToNodes != null) {
|
||||
addAttributesToNodesToProto();
|
||||
}
|
||||
}
|
||||
|
||||
private void mergeLocalToProto() {
|
||||
if (viaProto) {
|
||||
maybeInitBuilder();
|
||||
}
|
||||
mergeLocalToBuilder();
|
||||
proto = builder.build();
|
||||
viaProto = true;
|
||||
}
|
||||
|
||||
public GetAttributesToNodesResponseProto getProto() {
|
||||
mergeLocalToProto();
|
||||
proto = viaProto ? proto : builder.build();
|
||||
viaProto = true;
|
||||
return proto;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
assert false : "hashCode not designed";
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
if (other == null) {
|
||||
return false;
|
||||
}
|
||||
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAttributeToNodes(Map<NodeAttribute, Set<String>> map) {
|
||||
initAttributesToNodes();
|
||||
attributesToNodes.clear();
|
||||
attributesToNodes.putAll(map);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<NodeAttribute, Set<String>> getAttributesToNodes() {
|
||||
initAttributesToNodes();
|
||||
return this.attributesToNodes;
|
||||
}
|
||||
}
|
@ -0,0 +1,75 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||
|
||||
import com.google.protobuf.TextFormat;
|
||||
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesRequestProto;
|
||||
|
||||
/**
|
||||
* Request to get cluster node attributes.
|
||||
*/
|
||||
@Private
|
||||
@Unstable
|
||||
public class GetClusterNodeAttributesRequestPBImpl
|
||||
extends GetClusterNodeAttributesRequest {
|
||||
|
||||
private GetClusterNodeAttributesRequestProto proto =
|
||||
GetClusterNodeAttributesRequestProto.getDefaultInstance();
|
||||
private GetClusterNodeAttributesRequestProto.Builder builder = null;
|
||||
private boolean viaProto = false;
|
||||
|
||||
public GetClusterNodeAttributesRequestPBImpl() {
|
||||
builder = GetClusterNodeAttributesRequestProto.newBuilder();
|
||||
}
|
||||
|
||||
public GetClusterNodeAttributesRequestPBImpl(
|
||||
GetClusterNodeAttributesRequestProto proto) {
|
||||
this.proto = proto;
|
||||
viaProto = true;
|
||||
}
|
||||
|
||||
public GetClusterNodeAttributesRequestProto getProto() {
|
||||
proto = viaProto ? proto : builder.build();
|
||||
viaProto = true;
|
||||
return proto;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return getProto().hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
if (other == null) {
|
||||
return false;
|
||||
}
|
||||
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return TextFormat.shortDebugString(getProto());
|
||||
}
|
||||
}
|
@ -0,0 +1,156 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||
|
||||
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Cluster node attributes response.
|
||||
*/
|
||||
@Private
|
||||
@Unstable
|
||||
public class GetClusterNodeAttributesResponsePBImpl
|
||||
extends GetClusterNodeAttributesResponse {
|
||||
|
||||
private GetClusterNodeAttributesResponseProto proto =
|
||||
GetClusterNodeAttributesResponseProto.getDefaultInstance();
|
||||
private GetClusterNodeAttributesResponseProto.Builder builder = null;
|
||||
private Set<NodeAttribute> updatedNodeAttributes;
|
||||
private boolean viaProto = false;
|
||||
|
||||
public GetClusterNodeAttributesResponsePBImpl() {
|
||||
builder = GetClusterNodeAttributesResponseProto.newBuilder();
|
||||
}
|
||||
|
||||
public GetClusterNodeAttributesResponsePBImpl(
|
||||
GetClusterNodeAttributesResponseProto proto) {
|
||||
this.proto = proto;
|
||||
viaProto = true;
|
||||
}
|
||||
|
||||
public synchronized GetClusterNodeAttributesResponseProto getProto() {
|
||||
mergeLocalToProto();
|
||||
proto = viaProto ? proto : builder.build();
|
||||
viaProto = true;
|
||||
return proto;
|
||||
}
|
||||
|
||||
private void mergeLocalToProto() {
|
||||
if (viaProto) {
|
||||
maybeInitBuilder();
|
||||
}
|
||||
mergeLocalToBuilder();
|
||||
proto = builder.build();
|
||||
viaProto = true;
|
||||
}
|
||||
|
||||
private void mergeLocalToBuilder() {
|
||||
if (this.updatedNodeAttributes != null) {
|
||||
addNodeAttributesToProto();
|
||||
}
|
||||
}
|
||||
|
||||
private void addNodeAttributesToProto() {
|
||||
maybeInitBuilder();
|
||||
builder.clearNodeAttributes();
|
||||
List<NodeAttributeProto> protoList = new ArrayList<>();
|
||||
for (NodeAttribute r : this.updatedNodeAttributes) {
|
||||
protoList.add(convertToProtoFormat(r));
|
||||
}
|
||||
builder.addAllNodeAttributes(protoList);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
if (other == null) {
|
||||
return false;
|
||||
}
|
||||
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
assert false : "hashCode not designed";
|
||||
return 0;
|
||||
}
|
||||
|
||||
private void maybeInitBuilder() {
|
||||
if (viaProto || builder == null) {
|
||||
builder = GetClusterNodeAttributesResponseProto.newBuilder(proto);
|
||||
}
|
||||
viaProto = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void setNodeAttributes(Set<NodeAttribute> attributes) {
|
||||
maybeInitBuilder();
|
||||
this.updatedNodeAttributes = new HashSet<>();
|
||||
if (attributes == null) {
|
||||
builder.clearNodeAttributes();
|
||||
return;
|
||||
}
|
||||
this.updatedNodeAttributes.addAll(attributes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Set<NodeAttribute> getNodeAttributes() {
|
||||
if (this.updatedNodeAttributes != null) {
|
||||
return this.updatedNodeAttributes;
|
||||
}
|
||||
initLocalNodeAttributes();
|
||||
return this.updatedNodeAttributes;
|
||||
}
|
||||
|
||||
private void initLocalNodeAttributes() {
|
||||
YarnServiceProtos.GetClusterNodeAttributesResponseProtoOrBuilder p =
|
||||
viaProto ? proto : builder;
|
||||
List<NodeAttributeProto> attributesProtoList = p.getNodeAttributesList();
|
||||
this.updatedNodeAttributes = new HashSet<>();
|
||||
for (NodeAttributeProto r : attributesProtoList) {
|
||||
this.updatedNodeAttributes.add(convertFromProtoFormat(r));
|
||||
}
|
||||
}
|
||||
|
||||
private NodeAttribute convertFromProtoFormat(NodeAttributeProto p) {
|
||||
return new NodeAttributePBImpl(p);
|
||||
}
|
||||
|
||||
private NodeAttributeProto convertToProtoFormat(NodeAttribute t) {
|
||||
return ((NodeAttributePBImpl) t).getProto();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getProto().toString();
|
||||
}
|
||||
}
|
@ -88,12 +88,13 @@ public abstract Set<NodeAttribute> getClusterNodeAttributes(
|
||||
|
||||
/**
|
||||
* Given a attribute set, return what all Nodes have attribute mapped to it.
|
||||
* If the attributes set is null or empty, all attributes mapping are
|
||||
* returned.
|
||||
*
|
||||
* @return a Map, of attribute to set of hostnames
|
||||
* @return a Map of attributes to set of hostnames.
|
||||
*/
|
||||
//TODO need to handle as part of REST patch.
|
||||
/* public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||
Set<NodeAttribute> attributes);*/
|
||||
public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||
Set<NodeAttribute> attributes);
|
||||
|
||||
/**
|
||||
* NodeAttribute to AttributeValue Map.
|
||||
|
@ -53,8 +53,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl;
|
||||
@ -1259,4 +1263,28 @@ public void testNodesToAttributesMappingRequestPBImpl() throws Exception {
|
||||
validatePBImplRecord(NodesToAttributesMappingRequestPBImpl.class,
|
||||
NodesToAttributesMappingRequestProto.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAttributesToNodesRequestPBImpl() throws Exception {
|
||||
validatePBImplRecord(GetAttributesToNodesRequestPBImpl.class,
|
||||
YarnServiceProtos.GetAttributesToNodesRequestProto.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAttributesToNodesResponsePBImpl() throws Exception {
|
||||
validatePBImplRecord(GetAttributesToNodesResponsePBImpl.class,
|
||||
YarnServiceProtos.GetAttributesToNodesResponseProto.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetClusterNodeAttributesRequestPBImpl() throws Exception {
|
||||
validatePBImplRecord(GetClusterNodeAttributesRequestPBImpl.class,
|
||||
YarnServiceProtos.GetClusterNodeAttributesRequestProto.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetClusterNodeAttributesResponsePBImpl() throws Exception {
|
||||
validatePBImplRecord(GetClusterNodeAttributesResponsePBImpl.class,
|
||||
YarnServiceProtos.GetClusterNodeAttributesResponseProto.class);
|
||||
}
|
||||
}
|
||||
|
@ -53,8 +53,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
@ -895,6 +899,19 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||
GetClusterNodeAttributesRequest request)
|
||||
throws YarnException, IOException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public NodesToAttributesMappingResponse mapAttributesToNodes(NodesToAttributesMappingRequest request)
|
||||
throws YarnException, IOException {
|
||||
|
@ -66,8 +66,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
@ -127,6 +131,7 @@
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||
import org.apache.hadoop.yarn.api.records.Priority;
|
||||
@ -148,6 +153,7 @@
|
||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||
import org.apache.hadoop.yarn.ipc.RPCUtil;
|
||||
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
|
||||
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
|
||||
@ -1838,6 +1844,30 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
||||
return response;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||
NodeAttributesManager attributesManager =
|
||||
rmContext.getNodeAttributesManager();
|
||||
GetAttributesToNodesResponse response = GetAttributesToNodesResponse
|
||||
.newInstance(attributesManager
|
||||
.getAttributesToNodes(request.getNodeAttributes()));
|
||||
return response;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||
GetClusterNodeAttributesRequest request)
|
||||
throws YarnException, IOException {
|
||||
NodeAttributesManager attributesManager =
|
||||
rmContext.getNodeAttributesManager();
|
||||
Set<NodeAttribute> attributes =
|
||||
attributesManager.getClusterNodeAttributes(null);
|
||||
GetClusterNodeAttributesResponse response =
|
||||
GetClusterNodeAttributesResponse.newInstance(attributes);
|
||||
return response;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public void setDisplayPerUserApps(boolean displayPerUserApps) {
|
||||
this.filterAppsByUser = displayPerUserApps;
|
||||
|
@ -36,7 +36,7 @@
|
||||
import java.util.List;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
@ -367,19 +367,26 @@ public Set<NodeAttribute> getClusterNodeAttributes(Set<String> prefix) {
|
||||
return attributes;
|
||||
}
|
||||
|
||||
// TODO need to handle as part of REST patch.
|
||||
/*
|
||||
* @Override public Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||
* Set<NodeAttribute> attributes) { try { readLock.lock(); boolean
|
||||
* fetchAllAttributes = (attributes == null || attributes.isEmpty());
|
||||
* Map<NodeAttribute, Set<String>> attributesToNodes = new HashMap<>(); for
|
||||
* (Entry<NodeAttribute, RMAttributeNodeLabel> attributeEntry :
|
||||
* attributeCollections .entrySet()) { if (fetchAllAttributes ||
|
||||
* attributes.contains(attributeEntry.getKey())) {
|
||||
* attributesToNodes.put(attributeEntry.getKey(),
|
||||
* attributeEntry.getValue().getAssociatedNodeIds()); } } return
|
||||
* attributesToNodes; } finally { readLock.unlock(); } }
|
||||
*/
|
||||
@Override
|
||||
public Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||
Set<NodeAttribute> attributes) {
|
||||
try {
|
||||
readLock.lock();
|
||||
boolean fetchAllAttributes = (attributes == null || attributes.isEmpty());
|
||||
Map<NodeAttribute, Set<String>> attributesToNodes = new HashMap<>();
|
||||
for (Entry<NodeAttribute, RMNodeAttribute> attributeEntry :
|
||||
clusterAttributes.entrySet()) {
|
||||
if (fetchAllAttributes || attributes
|
||||
.contains(attributeEntry.getKey())) {
|
||||
attributesToNodes.put(attributeEntry.getKey(),
|
||||
attributeEntry.getValue().getAssociatedNodeIds());
|
||||
}
|
||||
}
|
||||
return attributesToNodes;
|
||||
} finally {
|
||||
readLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public Resource getResourceByAttribute(NodeAttribute attribute) {
|
||||
try {
|
||||
|
@ -18,6 +18,13 @@
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager;
|
||||
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||
import org.apache.hadoop.yarn.api.records.NodeAttributeType;
|
||||
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Matchers.any;
|
||||
@ -1999,6 +2006,125 @@ protected ClientRMService createClientRMService() {
|
||||
rm.close();
|
||||
}
|
||||
|
||||
@Test(timeout = 120000)
|
||||
public void testGetClusterNodeAttributes() throws IOException, YarnException {
|
||||
MockRM rm = new MockRM() {
|
||||
protected ClientRMService createClientRMService() {
|
||||
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
||||
this.applicationACLsManager, this.queueACLsManager,
|
||||
this.getRMContext().getRMDelegationTokenSecretManager());
|
||||
}
|
||||
};
|
||||
rm.start();
|
||||
|
||||
NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager();
|
||||
NodeId host1 = NodeId.newInstance("host1", 0);
|
||||
NodeId host2 = NodeId.newInstance("host2", 0);
|
||||
NodeAttribute gpu = NodeAttribute
|
||||
.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
|
||||
NodeAttributeType.STRING, "nvida");
|
||||
NodeAttribute os = NodeAttribute
|
||||
.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS",
|
||||
NodeAttributeType.STRING, "windows64");
|
||||
NodeAttribute docker = NodeAttribute
|
||||
.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER",
|
||||
NodeAttributeType.STRING, "docker0");
|
||||
Map<String, Set<NodeAttribute>> nodes = new HashMap<>();
|
||||
nodes.put(host1.getHost(), ImmutableSet.of(gpu, os));
|
||||
nodes.put(host2.getHost(), ImmutableSet.of(docker));
|
||||
mgr.addNodeAttributes(nodes);
|
||||
// Create a client.
|
||||
Configuration conf = new Configuration();
|
||||
YarnRPC rpc = YarnRPC.create(conf);
|
||||
InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
|
||||
LOG.info("Connecting to ResourceManager at " + rmAddress);
|
||||
ApplicationClientProtocol client = (ApplicationClientProtocol) rpc
|
||||
.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
|
||||
|
||||
GetClusterNodeAttributesRequest request =
|
||||
GetClusterNodeAttributesRequest.newInstance();
|
||||
GetClusterNodeAttributesResponse response =
|
||||
client.getClusterNodeAttributes(request);
|
||||
Set<NodeAttribute> attributes = response.getNodeAttributes();
|
||||
Assert.assertEquals("Size not correct", 3, attributes.size());
|
||||
Assert.assertTrue(attributes.contains(gpu));
|
||||
Assert.assertTrue(attributes.contains(os));
|
||||
Assert.assertTrue(attributes.contains(docker));
|
||||
rpc.stopProxy(client, conf);
|
||||
rm.close();
|
||||
}
|
||||
|
||||
@Test(timeout = 120000)
|
||||
public void testGetAttributesToNodes() throws IOException, YarnException {
|
||||
MockRM rm = new MockRM() {
|
||||
protected ClientRMService createClientRMService() {
|
||||
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
||||
this.applicationACLsManager, this.queueACLsManager,
|
||||
this.getRMContext().getRMDelegationTokenSecretManager());
|
||||
}
|
||||
};
|
||||
rm.start();
|
||||
|
||||
NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager();
|
||||
String node1 = "host1";
|
||||
String node2 = "host2";
|
||||
NodeAttribute gpu = NodeAttribute
|
||||
.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
|
||||
NodeAttributeType.STRING, "nvida");
|
||||
NodeAttribute os = NodeAttribute
|
||||
.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS",
|
||||
NodeAttributeType.STRING, "windows64");
|
||||
NodeAttribute docker = NodeAttribute
|
||||
.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER",
|
||||
NodeAttributeType.STRING, "docker0");
|
||||
NodeAttribute dist = NodeAttribute
|
||||
.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION",
|
||||
NodeAttributeType.STRING, "3_0_2");
|
||||
Map<String, Set<NodeAttribute>> nodes = new HashMap<>();
|
||||
nodes.put(node1, ImmutableSet.of(gpu, os, dist));
|
||||
nodes.put(node2, ImmutableSet.of(docker, dist));
|
||||
mgr.addNodeAttributes(nodes);
|
||||
// Create a client.
|
||||
Configuration conf = new Configuration();
|
||||
YarnRPC rpc = YarnRPC.create(conf);
|
||||
InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
|
||||
LOG.info("Connecting to ResourceManager at " + rmAddress);
|
||||
ApplicationClientProtocol client = (ApplicationClientProtocol) rpc
|
||||
.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
|
||||
|
||||
GetAttributesToNodesRequest request =
|
||||
GetAttributesToNodesRequest.newInstance();
|
||||
GetAttributesToNodesResponse response =
|
||||
client.getAttributesToNodes(request);
|
||||
Map<NodeAttribute, Set<String>> attrs = response.getAttributesToNodes();
|
||||
Assert.assertEquals(response.getAttributesToNodes().size(), 4);
|
||||
Assert.assertEquals(attrs.get(dist).size(), 2);
|
||||
Assert.assertEquals(attrs.get(os).size(), 1);
|
||||
Assert.assertEquals(attrs.get(gpu).size(), 1);
|
||||
Assert.assertTrue(attrs.get(dist).contains(node1));
|
||||
Assert.assertTrue(attrs.get(dist).contains(node2));
|
||||
Assert.assertTrue(attrs.get(docker).contains(node2));
|
||||
|
||||
GetAttributesToNodesRequest request2 =
|
||||
GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker));
|
||||
GetAttributesToNodesResponse response2 =
|
||||
client.getAttributesToNodes(request2);
|
||||
Map<NodeAttribute, Set<String>> attrs2 = response2.getAttributesToNodes();
|
||||
Assert.assertEquals(response2.getAttributesToNodes().size(), 1);
|
||||
Assert.assertTrue(attrs.get(docker).contains(node2));
|
||||
|
||||
GetAttributesToNodesRequest request3 =
|
||||
GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker, os));
|
||||
GetAttributesToNodesResponse response3 =
|
||||
client.getAttributesToNodes(request3);
|
||||
Map<NodeAttribute, Set<String>> attrs3 = response3.getAttributesToNodes();
|
||||
Assert.assertEquals(response3.getAttributesToNodes().size(), 2);
|
||||
Assert.assertTrue(attrs.get(os).contains(node1));
|
||||
Assert.assertTrue(attrs.get(docker).contains(node2));
|
||||
rpc.stopProxy(client, conf);
|
||||
rm.close();
|
||||
}
|
||||
|
||||
@Test(timeout = 120000)
|
||||
public void testUpdatePriorityAndKillAppWithZeroClusterResource()
|
||||
throws Exception {
|
||||
|
@ -39,8 +39,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
@ -327,6 +331,19 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
||||
return clientRMProxy.getResourceTypeInfo(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||
return clientRMProxy.getAttributesToNodes(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||
GetClusterNodeAttributesRequest request)
|
||||
throws YarnException, IOException {
|
||||
return clientRMProxy.getClusterNodeAttributes(request);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public void setRMClient(ApplicationClientProtocol clientRM) {
|
||||
this.clientRMProxy = clientRM;
|
||||
|
@ -55,8 +55,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
@ -826,4 +830,17 @@ public void shutdown() {
|
||||
executorService.shutdown();
|
||||
super.shutdown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||
throw new NotImplementedException("Code is not implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||
GetClusterNodeAttributesRequest request)
|
||||
throws YarnException, IOException {
|
||||
throw new NotImplementedException("Code is not implemented");
|
||||
}
|
||||
}
|
||||
|
@ -50,8 +50,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
@ -430,6 +434,21 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
||||
return pipeline.getRootInterceptor().getResourceTypeInfo(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||
RequestInterceptorChainWrapper pipeline = getInterceptorChain();
|
||||
return pipeline.getRootInterceptor().getAttributesToNodes(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||
GetClusterNodeAttributesRequest request)
|
||||
throws YarnException, IOException {
|
||||
RequestInterceptorChainWrapper pipeline = getInterceptorChain();
|
||||
return pipeline.getRootInterceptor().getClusterNodeAttributes(request);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected RequestInterceptorChainWrapper getInterceptorChain()
|
||||
throws IOException {
|
||||
|
@ -36,8 +36,12 @@
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||
@ -288,4 +292,17 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
||||
GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
|
||||
return getNextInterceptor().getResourceTypeInfo(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||
return getNextInterceptor().getAttributesToNodes(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||
GetClusterNodeAttributesRequest request)
|
||||
throws YarnException, IOException {
|
||||
return getNextInterceptor().getClusterNodeAttributes(request);
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user