diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 5c1b42f5ac..75ce2d60d7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -122,6 +122,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -1245,9 +1246,9 @@ private static long computeSlotMillis(TaskAttemptImpl taskAttempt) { int slotMemoryReq = taskAttempt.getMemoryRequired(taskAttempt.conf, taskType); - int minSlotMemSize = - taskAttempt.appContext.getClusterInfo().getMinContainerCapability() - .getMemory(); + int minSlotMemSize = taskAttempt.conf.getInt( + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); int simSlotsRequired = minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) slotMemoryReq diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 68d94268a0..2f857ead5a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -220,13 +220,9 @@ protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); if (this.clusterInfo != null) { - getContext().getClusterInfo().setMinContainerCapability( - this.clusterInfo.getMinContainerCapability()); getContext().getClusterInfo().setMaxContainerCapability( this.clusterInfo.getMaxContainerCapability()); } else { - getContext().getClusterInfo().setMinContainerCapability( - Resource.newInstance(1024, 1)); getContext().getClusterInfo().setMaxContainerCapability( Resource.newInstance(10240, 1)); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java index 34ad98bf39..7a579ca0c8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java @@ -204,8 +204,6 @@ protected AMRMProtocol createSchedulerProxy() { throws IOException { RegisterApplicationMasterResponse response = Records.newRecord(RegisterApplicationMasterResponse.class); - response.setMinimumResourceCapability(Resource.newInstance( - 1024, 1)); response.setMaximumResourceCapability(Resource.newInstance( 10240, 1)); return response; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 4bda49e5d8..a5dc3598c6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -1329,8 +1329,7 @@ private static AppContext createAppContext( when(context.getApplicationAttemptId()).thenReturn(appAttemptId); when(context.getJob(isA(JobId.class))).thenReturn(job); when(context.getClusterInfo()).thenReturn( - new ClusterInfo(Resource.newInstance(1024, 1), Resource.newInstance( - 10240, 1))); + new ClusterInfo(Resource.newInstance(10240, 1))); when(context.getEventHandler()).thenReturn(new EventHandler() { @Override public void handle(Event event) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java index 110cd86a33..1e7f3bfe75 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java @@ -1424,8 +1424,6 @@ private MapTaskImpl getMockMapTask(long clusterTimestamp, EventHandler eh) { when(minContainerRequirements.getMemory()).thenReturn(1000); ClusterInfo clusterInfo = mock(ClusterInfo.class); - when(clusterInfo.getMinContainerCapability()).thenReturn( - minContainerRequirements); AppContext appContext = mock(AppContext.class); when(appContext.getClusterInfo()).thenReturn(clusterInfo); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java index 8f1e6e9691..676cb20d06 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java @@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; import org.junit.Test; @@ -197,8 +198,9 @@ public void verifySlotMillis(int mapMemMb, int reduceMemMb, Configuration conf = new Configuration(); conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb); conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb); - app.setClusterInfo(new ClusterInfo(Resource - .newInstance(minContainerSize, 1), Resource.newInstance(10240, 1))); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + minContainerSize); + app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1))); Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); @@ -392,7 +394,6 @@ public void testContainerCleanedWhileRunning() throws Exception { ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(clusterInfo.getMinContainerCapability()).thenReturn(resource); when(resource.getMemory()).thenReturn(1024); TaskAttemptImpl taImpl = @@ -450,7 +451,6 @@ public void testContainerCleanedWhileCommitting() throws Exception { ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(clusterInfo.getMinContainerCapability()).thenReturn(resource); when(resource.getMemory()).thenReturn(1024); TaskAttemptImpl taImpl = @@ -511,7 +511,6 @@ public void testDoubleTooManyFetchFailure() throws Exception { ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(clusterInfo.getMinContainerCapability()).thenReturn(resource); when(resource.getMemory()).thenReturn(1024); TaskAttemptImpl taImpl = @@ -579,7 +578,6 @@ public void testAppDiognosticEventOnUnassignedTask() throws Exception { ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(clusterInfo.getMinContainerCapability()).thenReturn(resource); when(resource.getMemory()).thenReturn(1024); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, @@ -629,7 +627,6 @@ public void testAppDiognosticEventOnNewTask() throws Exception { ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(clusterInfo.getMinContainerCapability()).thenReturn(resource); when(resource.getMemory()).thenReturn(1024); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java index d0b82250b5..86cf98ece9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java @@ -117,8 +117,7 @@ private static AppContext createAppContext() { when(ctx.getApplicationAttemptId()).thenReturn(attemptId); when(ctx.getJob(isA(JobId.class))).thenReturn(job); when(ctx.getClusterInfo()).thenReturn( - new ClusterInfo(Resource.newInstance(1024, 1), Resource.newInstance( - 10240, 1))); + new ClusterInfo(Resource.newInstance(10240, 1))); when(ctx.getEventHandler()).thenReturn(eventHandler); return ctx; } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6348ee4a3e..a52ec9c4eb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -149,6 +149,9 @@ Release 2.1.0-beta - UNRELEASED YARN-821. Renamed setFinishApplicationStatus to setFinalApplicationStatus in FinishApplicationMasterRequest for consistency. (Jian He via vinodk) + YARN-787. Removed minimum resource from RegisterApplicationMasterResponse. + (tucu via acmurthy) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java index ca507d0128..58157105a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java @@ -51,24 +51,10 @@ public static RegisterApplicationMasterResponse newInstance( Map acls) { RegisterApplicationMasterResponse response = Records.newRecord(RegisterApplicationMasterResponse.class); - response.setMinimumResourceCapability(minCapability); response.setMaximumResourceCapability(maxCapability); response.setApplicationACLs(acls); return response; } - - /** - * Get the minimum capability for any {@link Resource} allocated by the - * ResourceManager in the cluster. - * @return minimum capability of allocated resources in the cluster - */ - @Public - @Stable - public abstract Resource getMinimumResourceCapability(); - - @Private - @Unstable - public abstract void setMinimumResourceCapability(Resource capability); /** * Get the maximum capability for any {@link Resource} allocated by the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java index aeeba997e3..cca989bd6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java @@ -42,7 +42,6 @@ public class RegisterApplicationMasterResponsePBImpl extends RegisterApplicationMasterResponseProto.Builder builder = null; boolean viaProto = false; - private Resource minimumResourceCapability; private Resource maximumResourceCapability; private Map applicationACLS = null; @@ -91,10 +90,6 @@ private void mergeLocalToProto() { } private void mergeLocalToBuilder() { - if (this.minimumResourceCapability != null) { - builder.setMinimumCapability( - convertToProtoFormat(this.minimumResourceCapability)); - } if (this.maximumResourceCapability != null) { builder.setMaximumCapability( convertToProtoFormat(this.maximumResourceCapability)); @@ -127,21 +122,6 @@ public Resource getMaximumResourceCapability() { return this.maximumResourceCapability; } - @Override - public Resource getMinimumResourceCapability() { - if (this.minimumResourceCapability != null) { - return this.minimumResourceCapability; - } - - RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasMinimumCapability()) { - return null; - } - - this.minimumResourceCapability = convertFromProtoFormat(p.getMinimumCapability()); - return this.minimumResourceCapability; - } - @Override public void setMaximumResourceCapability(Resource capability) { maybeInitBuilder(); @@ -151,16 +131,6 @@ public void setMaximumResourceCapability(Resource capability) { this.maximumResourceCapability = capability; } - @Override - public void setMinimumResourceCapability(Resource capability) { - maybeInitBuilder(); - if(minimumResourceCapability == null) { - builder.clearMinimumCapability(); - } - this.minimumResourceCapability = capability; - } - - @Override public Map getApplicationACLs() { initApplicationACLs(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index d697fb528d..84d5afb9c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -35,9 +35,8 @@ message RegisterApplicationMasterRequestProto { } message RegisterApplicationMasterResponseProto { - optional ResourceProto minimumCapability = 1; - optional ResourceProto maximumCapability = 2; - repeated ApplicationACLMapProto application_ACLs = 3; + optional ResourceProto maximumCapability = 1; + repeated ApplicationACLMapProto application_ACLs = 2; } message FinishApplicationMasterRequestProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 0c39f7a661..879214ad77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -463,22 +463,11 @@ public boolean run() throws YarnException, IOException { appMasterTrackingUrl); // Dump out information about cluster capability as seen by the // resource manager - int minMem = response.getMinimumResourceCapability().getMemory(); int maxMem = response.getMaximumResourceCapability().getMemory(); - LOG.info("Min mem capabililty of resources in this cluster " + minMem); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); - // A resource ask has to be atleast the minimum of the capability of the - // cluster, the value has to be a multiple of the min value and cannot - // exceed the max. - // If it is not an exact multiple of min, the RM will allocate to the - // nearest multiple of min - if (containerMemory < minMem) { - LOG.info("Container memory specified below min threshold of cluster." - + " Using min value." + ", specified=" + containerMemory + ", min=" - + minMem); - containerMemory = minMem; - } else if (containerMemory > maxMem) { + // A resource ask cannot exceed the max. + if (containerMemory > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java index 64c7ca7a66..7c75e2aaaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java @@ -22,27 +22,16 @@ import org.apache.hadoop.yarn.util.Records; public class ClusterInfo { - private Resource minContainerCapability; private Resource maxContainerCapability; public ClusterInfo() { - this.minContainerCapability = Records.newRecord(Resource.class); this.maxContainerCapability = Records.newRecord(Resource.class); } - public ClusterInfo(Resource minCapability, Resource maxCapability) { - this.minContainerCapability = minCapability; + public ClusterInfo(Resource maxCapability) { this.maxContainerCapability = maxCapability; } - public Resource getMinContainerCapability() { - return minContainerCapability; - } - - public void setMinContainerCapability(Resource minContainerCapability) { - this.minContainerCapability = minContainerCapability; - } - public Resource getMaxContainerCapability() { return maxContainerCapability; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 410fc8a1c4..3d96ced018 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -210,8 +210,6 @@ public RegisterApplicationMasterResponse registerApplicationMaster( // Pick up min/max resource from scheduler... RegisterApplicationMasterResponse response = recordFactory .newRecordInstance(RegisterApplicationMasterResponse.class); - response.setMinimumResourceCapability(rScheduler - .getMinimumResourceCapability()); response.setMaximumResourceCapability(rScheduler .getMaximumResourceCapability()); response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId)