From ac16400e1fb85a4186e5bf5bbc9cf204735ae74f Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Sat, 7 Jan 2017 08:23:06 -0800 Subject: [PATCH] YARN-6022. Revert changes of AbstractResourceRequest (Contributed by Wangda Tan via Daniel Templeton) --- .../api/records/AbstractResourceRequest.java | 55 ------------------- .../yarn/api/records/ResourceRequest.java | 19 ++++++- .../api/records/UpdateContainerRequest.java | 18 +++++- .../server/scheduler/SchedulerRequestKey.java | 1 - .../server/resourcemanager/RMAppManager.java | 2 +- .../server/resourcemanager/RMServerUtils.java | 2 +- .../scheduler/AbstractYarnScheduler.java | 7 +-- .../scheduler/SchedulerUtils.java | 18 +++--- .../scheduler/YarnScheduler.java | 6 +- .../scheduler/fair/FairScheduler.java | 5 +- 10 files changed, 55 insertions(+), 78 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java deleted file mode 100644 index 819a60786f..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -/** - * {@code AbstractResourceRequest} represents a generic resource request made - * by an application to the {@code ResourceManager}. - *

- * It includes: - *

- * - * @see Resource - */ -@Public -@Unstable -public abstract class AbstractResourceRequest { - - /** - * Set the Resource capability of the request - * @param capability Resource capability of the request - */ - @Public - @Stable - public abstract void setCapability(Resource capability); - - /** - * Get the Resource capability of the request. - * @return Resource capability of the request - */ - @Public - @Stable - public abstract Resource getCapability(); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index ce85b216fa..5bedc879ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -58,8 +58,7 @@ import org.apache.hadoop.yarn.util.Records; */ @Public @Stable -public abstract class ResourceRequest extends AbstractResourceRequest - implements Comparable{ +public abstract class ResourceRequest implements Comparable { @Public @Stable @@ -506,6 +505,22 @@ public abstract class ResourceRequest extends AbstractResourceRequest public void setAllocationRequestId(long allocationRequestID) { throw new UnsupportedOperationException(); } + + /** + * Set the Resource capability of the request. + * @param capability Resource capability of the request + */ + @Public + @Stable + public abstract void setCapability(Resource capability); + + /** + * Get the Resource capability of the request. + * @return Resource capability of the request + */ + @Public + @Stable + public abstract Resource getCapability(); @Override public int hashCode() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java index 925a7979f1..2060370508 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java @@ -62,7 +62,7 @@ import org.apache.hadoop.yarn.util.Records; */ @InterfaceAudience.Public @InterfaceStability.Unstable -public abstract class UpdateContainerRequest extends AbstractResourceRequest { +public abstract class UpdateContainerRequest { @InterfaceAudience.Public @InterfaceStability.Unstable @@ -143,6 +143,22 @@ public abstract class UpdateContainerRequest extends AbstractResourceRequest { @InterfaceStability.Unstable public abstract void setExecutionType(ExecutionType executionType); + /** + * Set the Resource capability of the request. + * @param capability Resource capability of the request + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract void setCapability(Resource capability); + + /** + * Get the Resource capability of the request. + * @return Resource capability of the request + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract Resource getCapability(); + @Override public int hashCode() { final int prime = 2153; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java index 36a91496d4..02539ba6f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.scheduler; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 9dd544e537..8232b88234 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -459,7 +459,7 @@ public class RMAppManager implements EventHandler, throw e; } - scheduler.normalizeRequest(amReq); + amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability())); return amReq; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index ebbeb0f272..94bfd58484 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -337,7 +337,7 @@ public class RMServerUtils { return false; } ResourceScheduler scheduler = rmContext.getScheduler(); - scheduler.normalizeRequest(request); + request.setCapability(scheduler.getNormalizedResource(request.getCapability())); return true; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index acfcde8c3d..89f9ffaf45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -36,7 +36,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -1044,8 +1043,8 @@ public abstract class AbstractYarnScheduler } @Override - public void normalizeRequest(AbstractResourceRequest ask) { - SchedulerUtils.normalizeRequest(ask, + public Resource getNormalizedResource(Resource requestedResource) { + return SchedulerUtils.getNormalizedResource(requestedResource, getResourceCalculator(), getMinimumResourceCapability(), getMaximumResourceCapability(), @@ -1059,7 +1058,7 @@ public abstract class AbstractYarnScheduler */ protected void normalizeRequests(List asks) { for (ResourceRequest ask: asks) { - normalizeRequest(ask); + ask.setCapability(getNormalizedResource(ask.getCapability())); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 5360665f47..148f023381 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import java.io.IOException; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -33,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; @@ -134,29 +134,33 @@ public class SchedulerUtils { * Utility method to normalize a resource request, by insuring that the * requested memory is a multiple of minMemory and is not zero. */ + @VisibleForTesting public static void normalizeRequest( ResourceRequest ask, ResourceCalculator resourceCalculator, Resource minimumResource, Resource maximumResource) { - normalizeRequest(ask, resourceCalculator, - minimumResource, maximumResource, minimumResource); + ask.setCapability( + getNormalizedResource(ask.getCapability(), resourceCalculator, + minimumResource, maximumResource, minimumResource)); } /** * Utility method to normalize a resource request, by insuring that the * requested memory is a multiple of increment resource and is not zero. + * + * @return normalized resource */ - public static void normalizeRequest( - AbstractResourceRequest ask, + public static Resource getNormalizedResource( + Resource ask, ResourceCalculator resourceCalculator, Resource minimumResource, Resource maximumResource, Resource incrementResource) { Resource normalized = Resources.normalize( - resourceCalculator, ask.getCapability(), minimumResource, + resourceCalculator, ask, minimumResource, maximumResource, incrementResource); - ask.setCapability(normalized); + return normalized; } private static void normalizeNodeLabelExpressionInRequest( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 3185dc107f..5579efd763 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -381,7 +380,8 @@ public interface YarnScheduler extends EventHandler { /** * Normalize a resource request. * - * @param request the resource request to be normalized + * @param requestedResource the resource to be normalized + * @return the normalized resource */ - void normalizeRequest(AbstractResourceRequest request); + Resource getNormalizedResource(Resource requestedResource); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 78dfe61297..6bbccc6d30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -799,8 +798,8 @@ public class FairScheduler extends } @Override - public void normalizeRequest(AbstractResourceRequest ask) { - SchedulerUtils.normalizeRequest(ask, + public Resource getNormalizedResource(Resource requestedResource) { + return SchedulerUtils.getNormalizedResource(requestedResource, DOMINANT_RESOURCE_CALCULATOR, minimumAllocation, getMaximumResourceCapability(),