From 85826f6ca5a6d06b711a6805f7a1a6788852db05 Mon Sep 17 00:00:00 2001 From: Arun Suresh Date: Fri, 30 Dec 2016 08:46:10 -0800 Subject: [PATCH] YARN-6066. Opportunistic containers Minor fixes : API annotations, parameter name changes, checkstyles. (asuresh) (cherry picked from commit 4985217de453a04ddffd7b52644bdc8d153f753c) --- .../v2/app/rm/RMContainerAllocator.java | 20 +++++++++---------- .../apache/hadoop/mapreduce/MRJobConfig.java | 6 +++--- .../mapred/TestMROpportunisticMaps.java | 2 +- .../yarn/api/records/ContainerState.java | 2 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 9 +++++++++ .../site/markdown/OpportunisticContainers.md | 4 ++-- 6 files changed, 27 insertions(+), 16 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index 2b63b9e2b4..31bc380b41 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -237,10 +237,10 @@ protected void serviceInit(Configuration conf) throws Exception { // Init startTime to current time. If all goes well, it will be reset after // first attempt to contact RM. retrystartTime = System.currentTimeMillis(); - this.scheduledRequests.setNumOpportunisticMapsPer100( - conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE, - MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE)); - LOG.info(this.scheduledRequests.getNumOpportunisticMapsPer100() + + this.scheduledRequests.setNumOpportunisticMapsPercent( + conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT, + MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT)); + LOG.info(this.scheduledRequests.getNumOpportunisticMapsPercent() + "% of the mappers will be scheduled using OPPORTUNISTIC containers"); } @@ -1056,14 +1056,14 @@ class ScheduledRequests { final Map maps = new LinkedHashMap(); int mapsMod100 = 0; - int numOpportunisticMapsPer100 = 0; + int numOpportunisticMapsPercent = 0; - void setNumOpportunisticMapsPer100(int numMaps) { - this.numOpportunisticMapsPer100 = numMaps; + void setNumOpportunisticMapsPercent(int numMaps) { + this.numOpportunisticMapsPercent = numMaps; } - int getNumOpportunisticMapsPer100() { - return this.numOpportunisticMapsPer100; + int getNumOpportunisticMapsPercent() { + return this.numOpportunisticMapsPercent; } @VisibleForTesting @@ -1110,7 +1110,7 @@ void addMap(ContainerRequestEvent event) { maps.put(event.getAttemptID(), request); addContainerReq(request); } else { - if (mapsMod100 < numOpportunisticMapsPer100) { + if (mapsMod100 < numOpportunisticMapsPercent) { request = new ContainerRequest(event, PRIORITY_OPPORTUNISTIC_MAP, mapNodeLabelExpression); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 32bbe96ca8..7add7df43a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -1005,9 +1005,9 @@ public interface MRJobConfig { * requested by the AM will be opportunistic. If the total number of maps * for the job is less than 'x', then ALL maps will be OPPORTUNISTIC */ - public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE = - "mapreduce.job.num-opportunistic-maps-percentage"; - public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE = 0; + public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = + "mapreduce.job.num-opportunistic-maps-percent"; + public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0; /** * A comma-separated list of properties whose value will be redacted. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java index 462ff041dc..eed731ffd3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java @@ -145,7 +145,7 @@ private void runMergeTest(JobConf job, FileSystem fileSystem, int job.setNumReduceTasks(numReducers); // All OPPORTUNISTIC - job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE, percent); + job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT, percent); job.setInt("mapreduce.map.maxattempts", 1); job.setInt("mapreduce.reduce.maxattempts", 1); job.setInt("mapred.test.num_lines", numLines); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java index 4efd8c11eb..696fe062a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api.records; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Stable; /** @@ -37,5 +38,6 @@ public enum ContainerState { COMPLETE, /** Scheduled (awaiting resources) at the NM. */ + @InterfaceStability.Unstable SCHEDULED } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d0ade220ee..7dd5ce3324 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -314,6 +314,7 @@ public static boolean isAclEnabled(Configuration conf) { /** Setting that controls whether opportunistic container allocation * is enabled or not. */ + @Unstable public static final String OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED = RM_PREFIX + "opportunistic-container-allocation.enabled"; public static final boolean @@ -321,12 +322,14 @@ public static boolean isAclEnabled(Configuration conf) { /** Number of nodes to be used by the Opportunistic Container allocator for * dispatching containers during container allocation. */ + @Unstable public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED = RM_PREFIX + "opportunistic-container-allocation.nodes-used"; public static final int DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED = 10; /** Frequency for computing least loaded NMs. */ + @Unstable public static final String NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS = RM_PREFIX + "nm-container-queuing.sorting-nodes-interval-ms"; public static final long @@ -334,6 +337,7 @@ public static boolean isAclEnabled(Configuration conf) { /** Comparator for determining node load for scheduling of opportunistic * containers. */ + @Unstable public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR = RM_PREFIX + "nm-container-queuing.load-comparator"; public static final String DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR = @@ -341,6 +345,7 @@ public static boolean isAclEnabled(Configuration conf) { /** Value of standard deviation used for calculation of queue limit * thresholds. */ + @Unstable public static final String NM_CONTAINER_QUEUING_LIMIT_STDEV = RM_PREFIX + "nm-container-queuing.queue-limit-stdev"; public static final float DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV = @@ -349,6 +354,7 @@ public static boolean isAclEnabled(Configuration conf) { /** Min length of container queue at NodeManager. This is a cluster-wide * configuration that acts as the lower-bound of optimal queue length * calculated by the NodeQueueLoadMonitor */ + @Unstable public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH = RM_PREFIX + "nm-container-queuing.min-queue-length"; public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH = 5; @@ -356,17 +362,20 @@ public static boolean isAclEnabled(Configuration conf) { /** Max length of container queue at NodeManager. This is a cluster-wide * configuration that acts as the upper-bound of optimal queue length * calculated by the NodeQueueLoadMonitor */ + @Unstable public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH = RM_PREFIX + "nm-container-queuing.max-queue-length"; public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH = 15; /** Min queue wait time for a container at a NodeManager. */ + @Unstable public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS = RM_PREFIX + "nm-container-queuing.min-queue-wait-time-ms"; public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS = 10; /** Max queue wait time for a container queue at a NodeManager. */ + @Unstable public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS = RM_PREFIX + "nm-container-queuing.max-queue-wait-time-ms"; public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md index ac26d886d0..223930e1bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md @@ -72,10 +72,10 @@ By default, allocation of opportunistic containers is performed centrally throug The following command can be used to run a sample pi map-reduce job, executing 40% of mappers using opportunistic containers (substitute `3.0.0-alpha2-SNAPSHOT` below with the version of Hadoop you are using): ``` -$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar pi -Dmapreduce.job.num-opportunistic-maps-percentage="40" 50 100 +$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar pi -Dmapreduce.job.num-opportunistic-maps-percent="40" 50 100 ``` -By changing the value of `mapreduce.job.num-opportunistic-maps-percentage` in the above command, we can specify the percentage of mappers that can be executed through opportunistic containers. +By changing the value of `mapreduce.job.num-opportunistic-maps-percent` in the above command, we can specify the percentage of mappers that can be executed through opportunistic containers. ###Opportunistic Containers in Web UI