From c2032e251e86f2c05f6523a0c54f41ac4b6c78df Mon Sep 17 00:00:00 2001 From: Varun Vasudev Date: Sat, 22 Oct 2016 20:15:47 +0530 Subject: [PATCH] YARN-5708. Implement APIs to get resource profiles from the RM. Contributed by Varun Vasudev. --- .../hadoop/mapred/ResourceMgrDelegate.java | 13 ++ .../hadoop/mapred/TestClientRedirect.java | 17 +++ .../yarn/api/ApplicationClientProtocol.java | 37 +++++ .../GetAllResourceProfilesRequest.java | 35 +++++ .../GetAllResourceProfilesResponse.java | 60 ++++++++ .../GetResourceProfileRequest.java | 59 ++++++++ .../GetResourceProfileResponse.java | 68 +++++++++ .../yarn/api/records/ProfileCapability.java | 88 +++++++++++ .../proto/applicationclient_protocol.proto | 2 + .../src/main/proto/yarn_protos.proto | 15 ++ .../src/main/proto/yarn_service_protos.proto | 16 ++ .../hadoop/yarn/client/api/YarnClient.java | 25 +++ .../yarn/client/api/impl/YarnClientImpl.java | 19 +++ ...ApplicationClientProtocolPBClientImpl.java | 36 +++++ ...pplicationClientProtocolPBServiceImpl.java | 42 ++++++ .../GetAllResourceProfilesRequestPBImpl.java | 55 +++++++ .../GetAllResourceProfilesResponsePBImpl.java | 142 ++++++++++++++++++ .../pb/GetResourceProfileRequestPBImpl.java | 101 +++++++++++++ .../pb/GetResourceProfileResponsePBImpl.java | 112 ++++++++++++++ .../impl/pb/ProfileCapabilityPBImpl.java | 134 +++++++++++++++++ .../hadoop/yarn/api/TestPBImplRecords.java | 34 +++++ .../server/MockResourceManagerFacade.java | 16 ++ .../resourcemanager/ClientRMService.java | 41 +++++ .../server/resourcemanager/RMContext.java | 5 + .../server/resourcemanager/RMContextImpl.java | 12 ++ .../resourcemanager/ResourceManager.java | 9 ++ 26 files changed, 1193 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceProfilesRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceProfilesResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetResourceProfileRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetResourceProfileResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 62aa497292..a365f80cc0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; @@ -517,4 +518,16 @@ public void killApplication(ApplicationId appId, String diagnostics) throws YarnException, IOException { client.killApplication(appId, diagnostics); } + + @Override + public Map getResourceProfiles() + throws YarnException, IOException { + return client.getResourceProfiles(); + } + + @Override + public Resource getResourceProfile(String profile) + throws YarnException, IOException { + return client.getResourceProfile(profile); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index 65eac65484..cc50be0a52 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -72,6 +72,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -104,6 +106,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; @@ -494,6 +498,19 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( throws YarnException, IOException { return null; } + + @Override + public GetAllResourceProfilesResponse getResourceProfiles( + GetAllResourceProfilesRequest request) + throws YarnException, IOException { + return null; + } + + @Override + public GetResourceProfileResponse getResourceProfile( + GetResourceProfileRequest request) throws YarnException, IOException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index 6d39366dcc..013579f40e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -65,6 +65,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -589,4 +593,37 @@ SignalContainerResponse signalToContainer( public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( UpdateApplicationTimeoutsRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by clients to get all the resource profiles that are + * available on the ResourceManager. + *

+ * @param request request to get all the resource profiles + * @return Response containing a map of the profile name to Resource + * capabilities + * @throws YarnException if resource profiles are not enabled on the RM + * @throws IOException in case of other errors + */ + @Public + @Unstable + GetAllResourceProfilesResponse getResourceProfiles( + GetAllResourceProfilesRequest request) throws YarnException, IOException; + + /** + *

+ * The interface to get the details for a specific resource profile. + *

+ * @param request request to get the details of a resource profile + * @return Response containing the details for a particular resource profile + * @throws YarnException if resource profiles are not enabled on the RM or + * the profile cannot be found + * @throws IOException in case of other errors + */ + @Public + @Unstable + GetResourceProfileResponse getResourceProfile( + GetResourceProfileRequest request) throws YarnException, IOException; + + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceProfilesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceProfilesRequest.java new file mode 100644 index 0000000000..0bb9bf805d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceProfilesRequest.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +/** + * Request class for getting all the resource profiles from the RM. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class GetAllResourceProfilesRequest { + + public static GetAllResourceProfilesRequest newInstance() { + return Records.newRecord(GetAllResourceProfilesRequest.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceProfilesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceProfilesResponse.java new file mode 100644 index 0000000000..547770890d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceProfilesResponse.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.Records; + +import java.util.Map; + +/** + * Response class for getting all the resource profiles from the RM. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class GetAllResourceProfilesResponse { + + public static GetAllResourceProfilesResponse newInstance() { + return Records.newRecord(GetAllResourceProfilesResponse.class); + } + + public abstract void setResourceProfiles(Map profiles); + + public abstract Map getResourceProfiles(); + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || !(other instanceof GetAllResourceProfilesResponse)) { + return false; + } + return ((GetAllResourceProfilesResponse) other).getResourceProfiles() + .equals(this.getResourceProfiles()); + } + + @Override + public int hashCode() { + return this.getResourceProfiles().hashCode(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetResourceProfileRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetResourceProfileRequest.java new file mode 100644 index 0000000000..3655be946d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetResourceProfileRequest.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +/** + * Request class for getting the details for a particular resource profile. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class GetResourceProfileRequest { + + public static GetResourceProfileRequest newInstance(String profile) { + GetResourceProfileRequest request = + Records.newRecord(GetResourceProfileRequest.class); + request.setProfileName(profile); + return request; + } + + public abstract void setProfileName(String profileName); + + public abstract String getProfileName(); + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || !(other instanceof GetResourceProfileRequest)) { + return false; + } + return this.getProfileName() + .equals(((GetResourceProfileRequest) other).getProfileName()); + } + + @Override + public int hashCode() { + return getProfileName().hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetResourceProfileResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetResourceProfileResponse.java new file mode 100644 index 0000000000..a010644a79 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetResourceProfileResponse.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.Records; + +/** + * Response class for getting the details for a particular resource profile. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class GetResourceProfileResponse { + + public static GetResourceProfileResponse newInstance() { + return Records.newRecord(GetResourceProfileResponse.class); + } + + /** + * Get the resources that will be allocated if the profile was used. + * + * @return the resources that will be allocated if the profile was used. + */ + public abstract Resource getResource(); + + /** + * Set the resources that will be allocated if the profile is used. + * + * @param r Set the resources that will be allocated if the profile is used. + */ + public abstract void setResource(Resource r); + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || !(other instanceof GetResourceProfileResponse)) { + return false; + } + return this.getResource() + .equals(((GetResourceProfileResponse) other).getResource()); + } + + @Override + public int hashCode() { + return getResource().hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java new file mode 100644 index 0000000000..0a93b89d3d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +/** + * Class to capture capability requirements when using resource profiles. The + * ProfileCapability is meant to be used as part of the ResourceRequest. A + * profile capability has two pieces - the resource profile name and the + * overrides. The resource profile specifies the name of the resource profile + * to be used and the capability override is the overrides desired on specific + * resource types. For example, you could use the "minimum" profile and set the + * memory in the capability override to 4096M. This implies that you wish for + * the resources specified in the "minimum" profile but with 4096M memory. The + * conversion from the ProfileCapability to the Resource class with the actual + * resource requirements will be done by the ResourceManager, which has the + * actual profile to Resource mapping. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class ProfileCapability { + + public static ProfileCapability newInstance(String profile, + Resource override) { + ProfileCapability obj = Records.newRecord(ProfileCapability.class); + obj.setProfileName(profile); + obj.setProfileCapabilityOverride(override); + return obj; + } + + public abstract String getProfileName(); + + public abstract Resource getProfileCapabilityOverride(); + + public abstract void setProfileName(String profileName); + + public abstract void setProfileCapabilityOverride(Resource r); + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || !(other instanceof ProfileCapability)) { + return false; + } + return ((ProfileCapability) other).getProfileName() + .equals(this.getProfileName()) && ((ProfileCapability) other) + .getProfileCapabilityOverride() + .equals(this.getProfileCapabilityOverride()); + } + + @Override + public int hashCode() { + final int prime = 2153; + int result = 2459; + String name = getProfileName(); + Resource override = getProfileCapabilityOverride(); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((override == null) ? 0 : override.hashCode()); + return result; + } + + @Override + public String toString() { + return "{ profile: " + this.getProfileName() + ", capabilityOverride: " + + this.getProfileCapabilityOverride() + " }"; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index ba79db09a6..aac6732f4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -61,4 +61,6 @@ service ApplicationClientProtocolService { rpc updateApplicationPriority (UpdateApplicationPriorityRequestProto) returns (UpdateApplicationPriorityResponseProto); rpc signalToContainer(SignalContainerRequestProto) returns (SignalContainerResponseProto); rpc updateApplicationTimeouts (UpdateApplicationTimeoutsRequestProto) returns (UpdateApplicationTimeoutsResponseProto); + rpc getResourceProfiles(GetAllResourceProfilesRequestProto) returns (GetAllResourceProfilesResponseProto); + rpc getResourceProfile(GetResourceProfileRequestProto) returns (GetResourceProfileResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index cf241f9e12..09f06f40f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -81,6 +81,15 @@ message ResourceOptionProto { optional int32 over_commit_timeout = 2; } +message ResourceProfileEntry { + required string name = 1; + required ResourceProto resources = 2; +} + +message ResourceProfilesProto { + repeated ResourceProfileEntry resource_profiles_map = 1; +} + message NodeResourceMapProto { optional NodeIdProto node_id = 1; optional ResourceOptionProto resource_option = 2; @@ -322,6 +331,11 @@ enum ExecutionTypeProto { //////////////////////////////////////////////////////////////////////// ////// From AM_RM_Protocol ///////////////////////////////////////////// //////////////////////////////////////////////////////////////////////// +message ProfileCapabilityProto { + required string profile = 1; + required ResourceProto profileCapabilityOverride = 2; +} + message ResourceRequestProto { optional PriorityProto priority = 1; optional string resource_name = 2; @@ -331,6 +345,7 @@ message ResourceRequestProto { optional string node_label_expression = 6; optional ExecutionTypeRequestProto execution_type_request = 7; optional int64 allocation_request_id = 8 [default = 0]; + optional ProfileCapabilityProto profile = 9; } message ExecutionTypeRequestProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 7a7f03503c..9a9be568a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -48,6 +48,7 @@ message RegisterApplicationMasterResponseProto { optional string queue = 5; repeated NMTokenProto nm_tokens_from_previous_attempts = 6; repeated SchedulerResourceTypes scheduler_resource_types = 7; + optional ResourceProfilesProto resource_profiles = 8; } message FinishApplicationMasterRequestProto { @@ -279,6 +280,21 @@ message UpdateApplicationTimeoutsResponseProto { repeated ApplicationUpdateTimeoutMapProto application_timeouts = 1; } +message GetAllResourceProfilesRequestProto { +} + +message GetAllResourceProfilesResponseProto { + required ResourceProfilesProto resource_profiles = 1; +} + +message GetResourceProfileRequestProto { + required string profile = 1; +} + +message GetResourceProfileResponseProto { + required ResourceProto resources = 1; +} + ////////////////////////////////////////////////////// /////// client_NM_Protocol /////////////////////////// ////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 8c68a31ffe..24d28871eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -855,4 +856,28 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( throw new UnsupportedOperationException("The sub-class extending " + YarnClient.class.getName() + " is expected to implement this !"); } + + /** + *

+ * Get the resource profiles available in the RM. + *

+ * @return a Map of the resource profile names to their capabilities + * @throws YarnException if resource profiles are not enabled + * @throws IOException in case of other errors + */ + public abstract Map getResourceProfiles() + throws YarnException, IOException; + + /** + *

+ * Get the details of a specific resource profile from the RM. + *

+ * @param profile the profile name + * @return the capabilities of the resource profile + * @throws YarnException if resource profiles are not enabled or the profile + * cannot be found + * @throws IOException in case of other others + */ + public abstract Resource getResourceProfile(String profile) + throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 19cb10b1d1..b22a6936a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -43,6 +43,7 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -70,6 +71,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; @@ -101,6 +103,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -937,4 +940,20 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( throws YarnException, IOException { return rmClient.updateApplicationTimeouts(request); } + + @Override + public Map getResourceProfiles() + throws YarnException, IOException { + GetAllResourceProfilesRequest request = + GetAllResourceProfilesRequest.newInstance(); + return rmClient.getResourceProfiles(request).getResourceProfiles(); + } + + @Override + public Resource getResourceProfile(String profile) + throws YarnException, IOException { + GetResourceProfileRequest request = + GetResourceProfileRequest.newInstance(profile); + return rmClient.getResourceProfile(request).getResource(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index ad7cb29608..4bad60d8f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -89,6 +89,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FailApplicationAttemptRequestPBImpl; @@ -147,6 +151,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.proto.YarnServiceProtos; @@ -619,4 +627,32 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( return null; } } + + @Override + public GetAllResourceProfilesResponse getResourceProfiles( + GetAllResourceProfilesRequest request) throws YarnException, IOException { + YarnServiceProtos.GetAllResourceProfilesRequestProto requestProto = + ((GetAllResourceProfilesRequestPBImpl) request).getProto(); + try { + return new GetAllResourceProfilesResponsePBImpl( + proxy.getResourceProfiles(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public GetResourceProfileResponse getResourceProfile( + GetResourceProfileRequest request) throws YarnException, IOException { + YarnServiceProtos.GetResourceProfileRequestProto requestProto = + ((GetResourceProfileRequestPBImpl) request).getProto(); + try { + return new GetResourceProfileResponsePBImpl( + proxy.getResourceProfile(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index 93ce6a343c..1c85030e5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FailApplicationAttemptRequestPBImpl; @@ -116,6 +118,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UpdateApplicationTimeoutsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.proto.YarnServiceProtos; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FailApplicationAttemptRequestProto; @@ -169,6 +175,10 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateApplicationTimeoutsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -631,4 +641,36 @@ public UpdateApplicationTimeoutsResponseProto updateApplicationTimeouts( throw new ServiceException(e); } } + + @Override + public GetAllResourceProfilesResponseProto getResourceProfiles( + RpcController controller, GetAllResourceProfilesRequestProto proto) + throws ServiceException { + GetAllResourceProfilesRequestPBImpl req = + new GetAllResourceProfilesRequestPBImpl(proto); + try { + GetAllResourceProfilesResponse resp = real.getResourceProfiles(req); + return ((GetAllResourceProfilesResponsePBImpl) resp).getProto(); + } catch (YarnException ye) { + throw new ServiceException(ye); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } + + @Override + public GetResourceProfileResponseProto getResourceProfile( + RpcController controller, GetResourceProfileRequestProto proto) + throws ServiceException { + GetResourceProfileRequestPBImpl req = + new GetResourceProfileRequestPBImpl(proto); + try { + GetResourceProfileResponse resp = real.getResourceProfile(req); + return ((GetResourceProfileResponsePBImpl) resp).getProto(); + } catch (YarnException ye) { + throw new ServiceException(ye); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesRequestPBImpl.java new file mode 100644 index 0000000000..ba06251f7a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesRequestPBImpl.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesRequestProto; + +/** + * Protobuf implementation class for GetAllResourceProfilesRequest. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GetAllResourceProfilesRequestPBImpl + extends GetAllResourceProfilesRequest { + + private GetAllResourceProfilesRequestProto proto = + GetAllResourceProfilesRequestProto.getDefaultInstance(); + private GetAllResourceProfilesRequestProto.Builder builder = null; + + private boolean viaProto = false; + + public GetAllResourceProfilesRequestPBImpl() { + builder = GetAllResourceProfilesRequestProto.newBuilder(); + } + + public GetAllResourceProfilesRequestPBImpl( + GetAllResourceProfilesRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetAllResourceProfilesRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java new file mode 100644 index 0000000000..bc4a9b6973 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProfilesProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProfileEntry; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Protobuf implementation class for the GetAllResourceProfilesResponse. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GetAllResourceProfilesResponsePBImpl + extends GetAllResourceProfilesResponse { + + private GetAllResourceProfilesResponseProto proto = + GetAllResourceProfilesResponseProto.getDefaultInstance(); + private GetAllResourceProfilesResponseProto.Builder builder = null; + private boolean viaProto = false; + private Map profiles; + + public GetAllResourceProfilesResponsePBImpl() { + builder = GetAllResourceProfilesResponseProto.newBuilder(); + } + + public GetAllResourceProfilesResponsePBImpl( + GetAllResourceProfilesResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetAllResourceProfilesResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetAllResourceProfilesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (profiles != null) { + addProfilesToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void addProfilesToProto() { + maybeInitBuilder(); + builder.clearResourceProfiles(); + if (profiles == null) { + return; + } + ResourceProfilesProto.Builder profilesBuilder = + ResourceProfilesProto.newBuilder(); + for (Map.Entry entry : profiles.entrySet()) { + ResourceProfileEntry.Builder profileEntry = + ResourceProfileEntry.newBuilder(); + profileEntry.setName(entry.getKey()); + profileEntry.setResources(convertToProtoFormat(entry.getValue())); + profilesBuilder.addResourceProfilesMap(profileEntry); + } + builder.setResourceProfiles(profilesBuilder.build()); + } + + public void setResourceProfiles(Map resourceProfiles) { + initResourceProfiles(); + profiles.clear(); + profiles.putAll(resourceProfiles); + } + + public Map getResourceProfiles() { + initResourceProfiles(); + return profiles; + } + + private void initResourceProfiles() { + if (profiles != null) { + return; + } + profiles = new HashMap<>(); + GetAllResourceProfilesResponseProtoOrBuilder p = viaProto ? proto : builder; + List profilesList = + p.getResourceProfiles().getResourceProfilesMapList(); + for (ResourceProfileEntry entry : profilesList) { + profiles.put(entry.getName(), new ResourcePBImpl(entry.getResources())); + } + } + + private ResourceProto convertToProtoFormat(Resource res) { + ResourcePBImpl r = new ResourcePBImpl(); + r.setMemorySize(res.getMemorySize()); + r.setVirtualCores(res.getVirtualCores()); + return r.getProto(); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileRequestPBImpl.java new file mode 100644 index 0000000000..926680bfb1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileRequestPBImpl.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; + +/** + * Protobuf implementation for the GetResourceProfileRequest class. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GetResourceProfileRequestPBImpl extends GetResourceProfileRequest { + + private GetResourceProfileRequestProto proto = + GetResourceProfileRequestProto.getDefaultInstance(); + private GetResourceProfileRequestProto.Builder builder = null; + private boolean viaProto = false; + + private String profile; + + public GetResourceProfileRequestPBImpl() { + builder = GetResourceProfileRequestProto.newBuilder(); + } + + public GetResourceProfileRequestPBImpl(GetResourceProfileRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetResourceProfileRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public void setProfileName(String profileName) { + this.profile = profileName; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (profile != null) { + builder.setProfile(profile); + } + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetResourceProfileRequestProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String getProfileName() { + if (this.profile != null) { + return profile; + } + GetResourceProfileRequestProtoOrBuilder protoOrBuilder = + viaProto ? proto : builder; + if (protoOrBuilder.hasProfile()) { + profile = protoOrBuilder.getProfile(); + } + return profile; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java new file mode 100644 index 0000000000..b82eaacf61 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; +import org.apache.hadoop.yarn.util.resource.Resources; + +/** + * Protobuf implementation for the GetResourceProfileResponse class. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GetResourceProfileResponsePBImpl + extends GetResourceProfileResponse { + + private GetResourceProfileResponseProto proto = + GetResourceProfileResponseProto.getDefaultInstance(); + private GetResourceProfileResponseProto.Builder builder = null; + private boolean viaProto = false; + + private Resource resource; + + public GetResourceProfileResponsePBImpl() { + builder = GetResourceProfileResponseProto.newBuilder(); + } + + public GetResourceProfileResponsePBImpl( + GetResourceProfileResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public Resource getResource() { + if (resource != null) { + return resource; + } + GetResourceProfileResponseProtoOrBuilder p = viaProto ? proto : builder; + if (p.hasResources()) { + resource = Resource.newInstance(p.getResources().getMemory(), + p.getResources().getVirtualCores()); + } + return resource; + } + + public void setResource(Resource r) { + resource = Resources.clone(r); + } + + public GetResourceProfileResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (resource != null) { + builder.setResources(convertToProtoFormat(resource)); + } + } + + private ResourceProto convertToProtoFormat(Resource res) { + ResourcePBImpl r = new ResourcePBImpl(); + r.setMemorySize(res.getMemorySize()); + r.setVirtualCores(res.getVirtualCores()); + return r.getProto(); + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetResourceProfileResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java new file mode 100644 index 0000000000..d6d564340f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ProfileCapability; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.proto.YarnProtos; +import org.apache.hadoop.yarn.proto.YarnProtos.ProfileCapabilityProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ProfileCapabilityProto; +import org.apache.hadoop.yarn.util.resource.Resources; + +/** + * Protobuf implementation for the ProfileCapability class. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class ProfileCapabilityPBImpl extends ProfileCapability { + + private ProfileCapabilityProto proto = + ProfileCapabilityProto.getDefaultInstance(); + private ProfileCapabilityProto.Builder builder; + + private boolean viaProto; + + private String profile; + private Resource profileCapabilityOverride; + + public ProfileCapabilityPBImpl() { + builder = ProfileCapabilityProto.newBuilder(); + } + + public ProfileCapabilityPBImpl(ProfileCapabilityProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public String getProfileName() { + if (profile != null) { + return profile; + } + ProfileCapabilityProtoOrBuilder p = viaProto ? proto : builder; + if (p.hasProfile()) { + profile = p.getProfile(); + } + return profile; + } + + @Override + public Resource getProfileCapabilityOverride() { + if (profileCapabilityOverride != null) { + return profileCapabilityOverride; + } + ProfileCapabilityProtoOrBuilder p = viaProto ? proto : builder; + if (p.hasProfileCapabilityOverride()) { + profileCapabilityOverride = + Resources.clone(new ResourcePBImpl(p.getProfileCapabilityOverride())); + } + return profileCapabilityOverride; + } + + @Override + public void setProfileName(String profileName) { + this.profile = profileName; + } + + @Override + public void setProfileCapabilityOverride(Resource r) { + this.profileCapabilityOverride = r; + } + + public ProfileCapabilityProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (profile != null) { + builder.setProfile(profile); + } + if (profileCapabilityOverride != null) { + builder.setProfileCapabilityOverride( + convertToProtoFormat(profileCapabilityOverride)); + } + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ProfileCapabilityProto.newBuilder(proto); + } + viaProto = false; + } + + private YarnProtos.ResourceProto convertToProtoFormat(Resource res) { + ResourcePBImpl r = new ResourcePBImpl(); + r.setMemorySize(res.getMemorySize()); + r.setVirtualCores(res.getVirtualCores()); + return r.getProto(); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 2747d7dddc..ce0f265d99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptReportRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptsRequestPBImpl; @@ -74,6 +75,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueUserAclsInfoRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueUserAclsInfoResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.KillApplicationRequestPBImpl; @@ -127,6 +130,7 @@ import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.PreemptionResourceRequest; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ProfileCapability; import org.apache.hadoop.yarn.api.records.QueueConfigurations; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueState; @@ -176,6 +180,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionResourceRequestPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ProfileCapabilityPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.QueueInfoPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.QueueUserACLInfoPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl; @@ -303,6 +308,10 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ProfileCapabilityProto; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl; @@ -363,6 +372,7 @@ public static void setup() throws Exception { generateByNewInstance(NodeReport.class); generateByNewInstance(Token.class); generateByNewInstance(NMToken.class); + generateByNewInstance(ProfileCapability.class); generateByNewInstance(ResourceRequest.class); generateByNewInstance(ApplicationAttemptReport.class); generateByNewInstance(ApplicationResourceUsageReport.class); @@ -1157,4 +1167,28 @@ public void testExecutionTypeRequestPBImpl() throws Exception { validatePBImplRecord(ExecutionTypeRequestPBImpl.class, ExecutionTypeRequestProto.class); } + + @Test + public void testGetAllResourceProfilesResponsePBImpl() throws Exception { + validatePBImplRecord(GetAllResourceProfilesResponsePBImpl.class, + GetAllResourceProfilesResponseProto.class); + } + + @Test + public void testGetResourceProfileRequestPBImpl() throws Exception { + validatePBImplRecord(GetResourceProfileRequestPBImpl.class, + GetResourceProfileRequestProto.class); + } + + @Test + public void testGetResourceProfileResponsePBImpl() throws Exception { + validatePBImplRecord(GetResourceProfileResponsePBImpl.class, + GetResourceProfileResponseProto.class); + } + + @Test + public void testProfileCapabilityPBImpl() throws Exception { + validatePBImplRecord(ProfileCapabilityPBImpl.class, + ProfileCapabilityProto.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index e33d7e1977..a5831be20b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -41,6 +41,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -73,6 +75,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; @@ -852,4 +856,16 @@ public String[] getGroupsForUser(String user) throws IOException { return new String[0]; } + + @Override + public GetAllResourceProfilesResponse getResourceProfiles( + GetAllResourceProfilesRequest request) throws YarnException, IOException { + return null; + } + + @Override + public GetResourceProfileResponse getResourceProfile( + GetResourceProfileRequest request) throws YarnException, IOException { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index df3889313e..beda9996dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -112,6 +112,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; @@ -145,6 +149,7 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInputValidator; @@ -210,6 +215,8 @@ public class ClientRMService extends AbstractService implements private static final EnumSet ACTIVE_APP_STATES = EnumSet.of( RMAppState.ACCEPTED, RMAppState.RUNNING); + private ResourceProfilesManager resourceProfilesManager; + public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, QueueACLsManager queueACLsManager, @@ -232,6 +239,7 @@ public ClientRMService(RMContext rmContext, YarnScheduler scheduler, this.reservationSystem = rmContext.getReservationSystem(); this.clock = clock; this.rValidator = new ReservationInputValidator(clock); + resourceProfilesManager = rmContext.getResourceProfilesManager(); } @Override @@ -1766,4 +1774,37 @@ private RMApp verifyUserAccessForRMApp(ApplicationId applicationId, return application; } + @Override + public GetAllResourceProfilesResponse getResourceProfiles( + GetAllResourceProfilesRequest request) throws YarnException, IOException { + GetAllResourceProfilesResponse response = + GetAllResourceProfilesResponse.newInstance(); + response.setResourceProfiles(getResourceProfiles()); + return response; + } + + @Override + public GetResourceProfileResponse getResourceProfile( + GetResourceProfileRequest request) throws YarnException, IOException { + Map profiles = getResourceProfiles(); + if (!profiles.containsKey(request.getProfileName())) { + throw new YarnException( + "Resource profile '" + request.getProfileName() + "' not found"); + } + GetResourceProfileResponse response = + GetResourceProfileResponse.newInstance(); + response.setResource( + resourceProfilesManager.getProfile(request.getProfileName())); + return response; + } + + private Map getResourceProfiles() throws YarnException { + boolean resourceProfilesEnabled = getConfig() + .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, + YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED); + if (!resourceProfilesEnabled) { + throw new YarnException("Resource profiles are not enabled"); + } + return resourceProfilesManager.getResourceProfiles(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 0ea9516567..6df31354d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetimeMonitor; @@ -159,4 +160,8 @@ void setRMDelegatedNodeLabelsUpdater( String getHAZookeeperConnectionState(); ResourceManager getResourceManager(); + + ResourceProfilesManager getResourceProfilesManager(); + + void setResourceProfilesManager(ResourceProfilesManager mgr); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index db2c585c03..921e5d5fe7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetimeMonitor; @@ -82,6 +83,8 @@ public class RMContextImpl implements RMContext { */ private RMActiveServiceContext activeServiceContext; + private ResourceProfilesManager resourceProfilesManager; + /** * Default constructor. To be used in conjunction with setter methods for * individual fields. @@ -551,5 +554,14 @@ public RMAppLifetimeMonitor getRMAppLifetimeMonitor() { return this.activeServiceContext.getRMAppLifetimeMonitor(); } + @Override + public ResourceProfilesManager getResourceProfilesManager() { + return this.resourceProfilesManager; + } + + @Override + public void setResourceProfilesManager(ResourceProfilesManager mgr) { + this.resourceProfilesManager = mgr; + } // Note: Read java doc before adding any services over here. } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 5333f25432..e478dc245f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -78,6 +78,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.reservation.AbstractReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManagerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -231,6 +233,13 @@ protected void serviceInit(Configuration conf) throws Exception { this.rmContext = new RMContextImpl(); rmContext.setResourceManager(this); + + // add resource profiles here because it's used by AbstractYarnScheduler + ResourceProfilesManager resourceProfilesManager = + new ResourceProfilesManagerImpl(); + resourceProfilesManager.init(conf); + rmContext.setResourceProfilesManager(resourceProfilesManager); + this.configurationProvider = ConfigurationProviderFactory.getConfigurationProvider(conf); this.configurationProvider.init(this.conf);