From 819224dcf9c683aa52f58633ac8e13663f1916d8 Mon Sep 17 00:00:00 2001 From: Jian He Date: Mon, 11 Jul 2016 22:36:20 -0700 Subject: [PATCH] YARN-5270. Solve miscellaneous issues caused by YARN-4844. Contributed by Wangda Tan --- .../v2/app/job/impl/TaskAttemptImpl.java | 3 +- .../v2/app/rm/RMContainerAllocator.java | 4 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 2 +- .../hadoop/mapreduce/TestTypeConverter.java | 4 +- .../org/apache/hadoop/mapred/YARNRunner.java | 2 +- .../org/apache/hadoop/yarn/sls/SLSRunner.java | 2 +- .../scheduler/ResourceSchedulerWrapper.java | 12 ++-- .../sls/scheduler/SLSCapacityScheduler.java | 12 ++-- .../hadoop/yarn/api/records/Resource.java | 47 ++++++++++----- .../src/main/proto/yarn_protos.proto | 2 +- .../TestApplicationClientProtocolOnHA.java | 2 +- .../api/records/impl/pb/ResourcePBImpl.java | 15 ++--- .../resource/DefaultResourceCalculator.java | 2 +- .../resource/DominantResourceCalculator.java | 8 +-- .../util/resource/ResourceCalculator.java | 17 +++++- .../hadoop/yarn/util/resource/Resources.java | 57 ++++++++++++------- .../yarn/util/resource/TestResources.java | 6 +- .../yarn/server/utils/BuilderUtils.java | 5 +- .../hadoop/yarn/TestYarnServerApiClasses.java | 2 +- .../amrmproxy/BaseAMRMProxyTest.java | 3 +- .../metrics/TestNodeManagerMetrics.java | 8 +-- .../scheduler/ClusterNodeTracker.java | 2 +- .../scheduler/QueueMetrics.java | 27 ++++----- .../SchedulerApplicationAttempt.java | 2 +- .../capacity/CapacityHeadroomProvider.java | 2 +- .../scheduler/fair/FSAppAttempt.java | 4 +- .../scheduler/fair/FSLeafQueue.java | 6 +- .../fair/policies/ComputeFairShares.java | 4 +- .../webapp/dao/ResourceInfo.java | 4 +- .../yarn/server/resourcemanager/MockAM.java | 3 +- .../server/resourcemanager/MockNodes.java | 6 +- .../yarn/server/resourcemanager/MockRM.java | 10 ++-- .../TestResourceTrackerService.java | 6 +- .../TestWorkPreservingRMRestart.java | 2 +- .../TestReservationSystemUtil.java | 2 +- .../scheduler/TestQueueMetrics.java | 6 +- .../security/TestDelegationTokenRenewer.java | 4 +- 37 files changed, 175 insertions(+), 130 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 0da3afb4cf..4305824070 100755 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -135,7 +135,6 @@ import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.Clock; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.RackResolver; import com.google.common.annotations.VisibleForTesting; @@ -667,7 +666,7 @@ public TaskAttemptImpl(TaskId taskId, int i, //TODO:create the resource reqt for this Task attempt this.resourceCapability = recordFactory.newRecordInstance(Resource.class); - this.resourceCapability.setMemory( + this.resourceCapability.setMemorySize( getMemoryRequired(conf, taskId.getTaskType())); this.resourceCapability.setVirtualCores( getCpuRequired(conf, taskId.getTaskType())); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index 217337ef6f..ecd75db35f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -389,7 +389,7 @@ protected synchronized void handleEvent(ContainerAllocatorEvent event) { } } // set the resources - reqEvent.getCapability().setMemory(mapResourceRequest.getMemorySize()); + reqEvent.getCapability().setMemorySize(mapResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( mapResourceRequest.getVirtualCores()); scheduledRequests.addMap(reqEvent);//maps are immediately scheduled @@ -417,7 +417,7 @@ protected synchronized void handleEvent(ContainerAllocatorEvent event) { } } // set the resources - reqEvent.getCapability().setMemory(reduceResourceRequest.getMemorySize()); + reqEvent.getCapability().setMemorySize(reduceResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( reduceResourceRequest.getVirtualCores()); if (reqEvent.getEarlierAttemptFailed()) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 6ba93e69c0..3472b1ffc1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -254,7 +254,7 @@ public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, // the job can reaches the final state when MRAppMaster shuts down. this.successfullyUnregistered.set(unregistered); this.assignedQueue = assignedQueue; - this.resource = Resource.newInstance(1234L, 2L); + this.resource = Resource.newInstance(1234L, 2); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java index 2027d01b70..629a246765 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java @@ -89,7 +89,7 @@ public void testFromYarn() throws Exception { ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); - r.setMemory(2048); + r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); @@ -128,7 +128,7 @@ public void testFromYarnApplicationReport() { ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); - r.setMemory(2048); + r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index b30641ebb2..4c6f0f3139 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -337,7 +337,7 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( // Setup resource requirements Resource capability = recordFactory.newRecordInstance(Resource.class); - capability.setMemory( + capability.setMemorySize( conf.getInt( MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB ) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index 3be8ff1330..c9c5c3847d 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -323,7 +323,7 @@ private void startAMFromSLSTraces(Resource containerResource, if (jsonTask.containsKey("container.memory")) { int containerMemory = Integer.parseInt( jsonTask.get("container.memory").toString()); - res.setMemory(containerMemory); + res.setMemorySize(containerMemory); } if (jsonTask.containsKey("container.vcores")) { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 014aa89415..393300ca2b 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -542,11 +542,11 @@ public Long getValue() { } ); metrics.register("variable.cluster.allocated.vcores", - new Gauge() { + new Gauge() { @Override - public Long getValue() { + public Integer getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return scheduler.getRootQueueMetrics().getAllocatedVirtualCores(); } @@ -566,11 +566,11 @@ public Long getValue() { } ); metrics.register("variable.cluster.available.vcores", - new Gauge() { + new Gauge() { @Override - public Long getValue() { + public Integer getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return scheduler.getRootQueueMetrics().getAvailableVirtualCores(); } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index 5cd53fe6af..1c3fa792ae 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -519,11 +519,11 @@ public Long getValue() { } ); metrics.register("variable.cluster.allocated.vcores", - new Gauge() { + new Gauge() { @Override - public Long getValue() { + public Integer getValue() { if(getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return getRootQueueMetrics().getAllocatedVirtualCores(); } @@ -543,11 +543,11 @@ public Long getValue() { } ); metrics.register("variable.cluster.available.vcores", - new Gauge() { + new Gauge() { @Override - public Long getValue() { + public Integer getValue() { if(getRootQueueMetrics() == null) { - return 0l; + return 0; } else { return getRootQueueMetrics().getAvailableVirtualCores(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index e2d847284a..0d31b6fd7d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -18,14 +18,14 @@ package org.apache.hadoop.yarn.api.records; -import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.util.Records; + /** *

Resource models a set of computer resources in the * cluster.

@@ -55,9 +55,18 @@ public abstract class Resource implements Comparable { @Public @Stable - public static Resource newInstance(long memory, long vCores) { + public static Resource newInstance(int memory, int vCores) { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); + resource.setVirtualCores(vCores); + return resource; + } + + @Public + @Stable + public static Resource newInstance(long memory, int vCores) { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } @@ -77,17 +86,31 @@ public static Resource newInstance(long memory, long vCores) { * Get memory of the resource. * @return memory of the resource */ - @Private - @Unstable - public abstract long getMemorySize(); - + @Public + @Stable + public long getMemorySize() { + throw new NotImplementedException( + "This method is implemented by ResourcePBImpl"); + } + + /** + * Set memory of the resource. + * @param memory memory of the resource + */ + @Public + @Deprecated + public abstract void setMemory(int memory); + /** * Set memory of the resource. * @param memory memory of the resource */ @Public @Stable - public abstract void setMemory(long memory); + public void setMemorySize(long memory) { + throw new NotImplementedException( + "This method is implemented by ResourcePBImpl"); + } /** @@ -103,10 +126,6 @@ public static Resource newInstance(long memory, long vCores) { @Public @Evolving public abstract int getVirtualCores(); - - @Public - @Unstable - public abstract long getVirtualCoresSize(); /** * Set number of virtual cpu cores of the resource. @@ -120,7 +139,7 @@ public static Resource newInstance(long memory, long vCores) { */ @Public @Evolving - public abstract void setVirtualCores(long vCores); + public abstract void setVirtualCores(int vCores); @Override public int hashCode() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c84f4e9032..6c337cf3a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -55,7 +55,7 @@ message ContainerIdProto { message ResourceProto { optional int64 memory = 1; - optional int64 virtual_cores = 2; + optional int32 virtual_cores = 2; } message ResourceUtilizationProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java index 8e00554ad6..c9fa91513a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java @@ -171,7 +171,7 @@ public void testSubmitApplicationOnHA() throws Exception { Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(10); + capability.setMemorySize(10); capability.setVirtualCores(1); appContext.setResource(capability); ApplicationId appId = client.submitApplication(appContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 937941ae54..6686696cdb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -67,24 +67,25 @@ public long getMemorySize() { } @Override - public void setMemory(long memory) { + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + setMemorySize(memory); + } + + @Override + public void setMemorySize(long memory) { maybeInitBuilder(); builder.setMemory(memory); } @Override public int getVirtualCores() { - return (int) getVirtualCoresSize(); - } - - @Override - public long getVirtualCoresSize() { ResourceProtoOrBuilder p = viaProto ? proto : builder; return p.getVirtualCores(); } @Override - public void setVirtualCores(long vCores) { + public void setVirtualCores(int vCores) { maybeInitBuilder(); builder.setVirtualCores(vCores); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index b98cf0c518..fe78de2d97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -56,7 +56,7 @@ public float ratio(Resource a, Resource b) { } @Override - public Resource divideAndCeil(Resource numerator, long denominator) { + public Resource divideAndCeil(Resource numerator, int denominator) { return Resources.createResource( divideAndCeil(numerator.getMemorySize(), denominator)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 8f97fab032..2695811fc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -142,7 +142,7 @@ public float ratio(Resource a, Resource b) { } @Override - public Resource divideAndCeil(Resource numerator, long denominator) { + public Resource divideAndCeil(Resource numerator, int denominator) { return Resources.createResource( divideAndCeil(numerator.getMemorySize(), denominator), divideAndCeil(numerator.getVirtualCores(), denominator) @@ -157,7 +157,7 @@ public Resource normalize(Resource r, Resource minimumResource, Math.max(r.getMemorySize(), minimumResource.getMemorySize()), stepFactor.getMemorySize()), maximumResource.getMemorySize()); - long normalizedCores = Math.min( + int normalizedCores = Math.min( roundUp( Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()), stepFactor.getVirtualCores()), @@ -189,7 +189,7 @@ public Resource multiplyAndNormalizeUp(Resource r, double by, roundUp( (int)Math.ceil(r.getMemorySize() * by), stepFactor.getMemorySize()), roundUp( - (int)Math.ceil(r.getVirtualCores() * by), + (int)Math.ceil(r.getVirtualCores() * by), stepFactor.getVirtualCores()) ); } @@ -203,7 +203,7 @@ public Resource multiplyAndNormalizeDown(Resource r, double by, stepFactor.getMemorySize() ), roundDown( - (int)(r.getVirtualCores() * by), + (int)(r.getVirtualCores() * by), stepFactor.getVirtualCores() ) ); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index 3fe8adecb8..a7df9c53c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -30,6 +30,13 @@ public abstract class ResourceCalculator { public abstract int compare(Resource clusterResource, Resource lhs, Resource rhs); + + public static int divideAndCeil(int a, int b) { + if (b == 0) { + return 0; + } + return (a + (b - 1)) / b; + } public static long divideAndCeil(long a, long b) { if (b == 0) { @@ -38,6 +45,10 @@ public static long divideAndCeil(long a, long b) { return (a + (b - 1)) / b; } + public static int roundUp(int a, int b) { + return divideAndCeil(a, b) * b; + } + public static long roundUp(long a, long b) { return divideAndCeil(a, b) * b; } @@ -46,6 +57,10 @@ public static long roundDown(long a, long b) { return (a / b) * b; } + public static int roundDown(int a, int b) { + return (a / b) * b; + } + /** * Compute the number of containers which can be allocated given * available and required resources. @@ -169,7 +184,7 @@ public abstract float divide( * @param denominator denominator * @return resultant resource */ - public abstract Resource divideAndCeil(Resource numerator, long denominator); + public abstract Resource divideAndCeil(Resource numerator, int denominator); /** * Check if a smaller resource can be contained by bigger resource. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index e944e1174d..760b0ea0ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -42,7 +42,13 @@ public long getMemorySize() { } @Override - public void setMemory(long memory) { + public void setMemorySize(long memory) { + throw new RuntimeException("NONE cannot be modified!"); + } + + @Override + @SuppressWarnings("deprecation") + public void setMemory(int memory) { throw new RuntimeException("NONE cannot be modified!"); } @@ -52,12 +58,7 @@ public int getVirtualCores() { } @Override - public long getVirtualCoresSize() { - return 0; - } - - @Override - public void setVirtualCores(long cores) { + public void setVirtualCores(int cores) { throw new RuntimeException("NONE cannot be modified!"); } @@ -86,7 +87,13 @@ public long getMemorySize() { } @Override - public void setMemory(long memory) { + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + throw new RuntimeException("UNBOUNDED cannot be modified!"); + } + + @Override + public void setMemorySize(long memory) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @@ -96,12 +103,7 @@ public int getVirtualCores() { } @Override - public long getVirtualCoresSize() { - return Long.MAX_VALUE; - } - - @Override - public void setVirtualCores(long cores) { + public void setVirtualCores(int cores) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @@ -109,20 +111,31 @@ public void setVirtualCores(long cores) { public int compareTo(Resource o) { long diff = Long.MAX_VALUE - o.getMemorySize(); if (diff == 0) { - diff = Long.MAX_VALUE - o.getVirtualCoresSize(); + diff = Integer.MAX_VALUE - o.getVirtualCores(); } return Long.signum(diff); } }; + public static Resource createResource(int memory) { + return createResource(memory, (memory > 0) ? 1 : 0); + } + + public static Resource createResource(int memory, int cores) { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); + resource.setVirtualCores(cores); + return resource; + } + public static Resource createResource(long memory) { return createResource(memory, (memory > 0) ? 1 : 0); } - public static Resource createResource(long memory, long cores) { + public static Resource createResource(long memory, int cores) { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(cores); return resource; } @@ -140,7 +153,7 @@ public static Resource clone(Resource res) { } public static Resource addTo(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemorySize() + rhs.getMemorySize()); + lhs.setMemorySize(lhs.getMemorySize() + rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores()); return lhs; } @@ -150,7 +163,7 @@ public static Resource add(Resource lhs, Resource rhs) { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemorySize() - rhs.getMemorySize()); + lhs.setMemorySize(lhs.getMemorySize() - rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); return lhs; } @@ -164,7 +177,7 @@ public static Resource negate(Resource resource) { } public static Resource multiplyTo(Resource lhs, double by) { - lhs.setMemory((int)(lhs.getMemorySize() * by)); + lhs.setMemorySize((long)(lhs.getMemorySize() * by)); lhs.setVirtualCores((int)(lhs.getVirtualCores() * by)); return lhs; } @@ -179,7 +192,7 @@ public static Resource multiply(Resource lhs, double by) { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - lhs.setMemory(lhs.getMemorySize() + (int)(rhs.getMemorySize() * by)); + lhs.setMemorySize(lhs.getMemorySize() + (long)(rhs.getMemorySize() * by)); lhs.setVirtualCores(lhs.getVirtualCores() + (int)(rhs.getVirtualCores() * by)); return lhs; @@ -197,7 +210,7 @@ public static Resource multiplyAndNormalizeDown( public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - out.setMemory((int)(lhs.getMemorySize() * by)); + out.setMemorySize((long)(lhs.getMemorySize() * by)); out.setVirtualCores((int)(lhs.getVirtualCores() * by)); return out; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index 7396adc2a3..057214bada 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -24,18 +24,18 @@ public class TestResources { - public Resource createResource(long memory, long vCores) { + public Resource createResource(long memory, int vCores) { return Resource.newInstance(memory, vCores); } @Test(timeout=1000) public void testCompareToWithUnboundedResource() { assertTrue(Resources.unbounded().compareTo( - createResource(Long.MAX_VALUE, Long.MAX_VALUE)) == 0); + createResource(Long.MAX_VALUE, Integer.MAX_VALUE)) == 0); assertTrue(Resources.unbounded().compareTo( createResource(Long.MAX_VALUE, 0)) > 0); assertTrue(Resources.unbounded().compareTo( - createResource(0, Long.MAX_VALUE)) > 0); + createResource(0, Integer.MAX_VALUE)) > 0); } @Test(timeout=1000) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 134f22de33..a049fb0d3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; @@ -442,9 +441,9 @@ public static ApplicationResourceUsageReport newApplicationResourceUsageReport( return report; } - public static Resource newResource(long memory, long vCores) { + public static Resource newResource(long memory, int vCores) { Resource resource = recordFactory.newRecordInstance(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index 475ce58578..be350e2651 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -201,7 +201,7 @@ public void testRegisterNodeManagerRequestPBImpl() { original.setHttpPort(8080); original.setNodeId(getNodeId()); Resource resource = recordFactory.newRecordInstance(Resource.class); - resource.setMemory(10000); + resource.setMemorySize(10000); resource.setVirtualCores(2); original.setResource(resource); RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java index ae0cbc3e96..f716d44328 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java @@ -39,7 +39,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; @@ -465,7 +464,7 @@ protected ResourceRequest createResourceRequest(String resource, pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(memory); + capability.setMemorySize(memory); capability.setVirtualCores(vCores); req.setCapability(capability); if (labelExpression != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java index c0210d5a0b..a08ee82e75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java @@ -33,16 +33,16 @@ public class TestNodeManagerMetrics { DefaultMetricsSystem.initialize("NodeManager"); NodeManagerMetrics metrics = NodeManagerMetrics.create(); Resource total = Records.newRecord(Resource.class); - total.setMemory(8*GiB); + total.setMemorySize(8*GiB); total.setVirtualCores(16); Resource resource = Records.newRecord(Resource.class); - resource.setMemory(512); //512MiB + resource.setMemorySize(512); //512MiB resource.setVirtualCores(2); Resource largerResource = Records.newRecord(Resource.class); - largerResource.setMemory(1024); + largerResource.setMemorySize(1024); largerResource.setVirtualCores(2); Resource smallerResource = Records.newRecord(Resource.class); - smallerResource.setMemory(256); + smallerResource.setMemorySize(256); smallerResource.setVirtualCores(1); metrics.addResource(total); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java index feb071fca8..413bbdf7db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java @@ -57,7 +57,7 @@ public class ClusterNodeTracker { // Max allocation private long maxNodeMemory = -1; - private long maxNodeVCores = -1; + private int maxNodeVCores = -1; private Resource configuredMaxAllocation; private boolean forceConfiguredMaxAllocation = true; private long configuredMaxAllocationWaitTime; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 69c3e8b877..4e364f70f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -61,7 +61,7 @@ public class QueueMetrics implements MetricsSource { @Metric("# of apps failed") MutableCounterInt appsFailed; @Metric("Allocated memory in MB") MutableGaugeLong allocatedMB; - @Metric("Allocated CPU in virtual cores") MutableGaugeLong allocatedVCores; + @Metric("Allocated CPU in virtual cores") MutableGaugeInt allocatedVCores; @Metric("# of allocated containers") MutableGaugeInt allocatedContainers; @Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated; @Metric("Aggregate # of allocated node-local containers") @@ -72,12 +72,12 @@ public class QueueMetrics implements MetricsSource { MutableCounterLong aggregateOffSwitchContainersAllocated; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; @Metric("Available memory in MB") MutableGaugeLong availableMB; - @Metric("Available CPU in virtual cores") MutableGaugeLong availableVCores; + @Metric("Available CPU in virtual cores") MutableGaugeInt availableVCores; @Metric("Pending memory allocation in MB") MutableGaugeLong pendingMB; - @Metric("Pending CPU allocation in virtual cores") MutableGaugeLong pendingVCores; + @Metric("Pending CPU allocation in virtual cores") MutableGaugeInt pendingVCores; @Metric("# of pending containers") MutableGaugeInt pendingContainers; @Metric("# of reserved memory in MB") MutableGaugeLong reservedMB; - @Metric("Reserved CPU in virtual cores") MutableGaugeLong reservedVCores; + @Metric("Reserved CPU in virtual cores") MutableGaugeInt reservedVCores; @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active applications") MutableGaugeInt activeApplications; @@ -364,7 +364,7 @@ public void incrPendingResources(String user, int containers, Resource res) { private void _incrPendingResources(int containers, Resource res) { pendingContainers.incr(containers); pendingMB.incr(res.getMemorySize() * containers); - pendingVCores.incr(res.getVirtualCoresSize() * containers); + pendingVCores.incr(res.getVirtualCores() * containers); } public void decrPendingResources(String user, int containers, Resource res) { @@ -381,7 +381,7 @@ public void decrPendingResources(String user, int containers, Resource res) { private void _decrPendingResources(int containers, Resource res) { pendingContainers.decr(containers); pendingMB.decr(res.getMemorySize() * containers); - pendingVCores.decr(res.getVirtualCoresSize() * containers); + pendingVCores.decr(res.getVirtualCores() * containers); } public void incrNodeTypeAggregations(String user, NodeType type) { @@ -409,7 +409,7 @@ public void allocateResources(String user, int containers, Resource res, aggregateContainersAllocated.incr(containers); allocatedMB.incr(res.getMemorySize() * containers); - allocatedVCores.incr(res.getVirtualCoresSize() * containers); + allocatedVCores.incr(res.getVirtualCores() * containers); if (decrPending) { _decrPendingResources(containers, res); } @@ -448,7 +448,7 @@ public void releaseResources(String user, int containers, Resource res) { allocatedContainers.decr(containers); aggregateContainersReleased.incr(containers); allocatedMB.decr(res.getMemorySize() * containers); - allocatedVCores.decr(res.getVirtualCoresSize() * containers); + allocatedVCores.decr(res.getVirtualCores() * containers); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.releaseResources(user, containers, res); @@ -561,14 +561,15 @@ public int getAppsFailed() { } public Resource getAllocatedResources() { - return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value()); + return BuilderUtils.newResource(allocatedMB.value(), + (int) allocatedVCores.value()); } public long getAllocatedMB() { return allocatedMB.value(); } - public long getAllocatedVirtualCores() { + public int getAllocatedVirtualCores() { return allocatedVCores.value(); } @@ -580,7 +581,7 @@ public long getAvailableMB() { return availableMB.value(); } - public long getAvailableVirtualCores() { + public int getAvailableVirtualCores() { return availableVCores.value(); } @@ -588,7 +589,7 @@ public long getPendingMB() { return pendingMB.value(); } - public long getPendingVirtualCores() { + public int getPendingVirtualCores() { return pendingVCores.value(); } @@ -600,7 +601,7 @@ public long getReservedMB() { return reservedMB.value(); } - public long getReservedVirtualCores() { + public int getReservedVirtualCores() { return reservedVCores.value(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index aae07708e6..b4a2639a1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -465,7 +465,7 @@ public synchronized void setHeadroom(Resource globalLimit) { public synchronized Resource getHeadroom() { // Corner case to deal with applications being slightly over-limit if (resourceLimit.getMemorySize() < 0) { - resourceLimit.setMemory(0); + resourceLimit.setMemorySize(0); } return resourceLimit; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java index 49892f7352..5605f18b77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java @@ -66,7 +66,7 @@ public Resource getHeadroom() { } // Corner case to deal with applications being slightly over-limit if (headroom.getMemorySize() < 0) { - headroom.setMemory(0); + headroom.setMemorySize(0); } return headroom; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 634f667cd5..1eead9a11c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -194,7 +194,7 @@ private void subtractResourcesOnBlacklistedNodes( node.getUnallocatedResource()); } if (availableResources.getMemorySize() < 0) { - availableResources.setMemory(0); + availableResources.setMemorySize(0); } if (availableResources.getVirtualCores() < 0) { availableResources.setVirtualCores(0); @@ -460,7 +460,7 @@ public void resetPreemptedResources() { } public void clearPreemptedResources() { - preemptedResources.setMemory(0); + preemptedResources.setMemorySize(0); preemptedResources.setVirtualCores(0); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 6723fd9e74..7c9b11edbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -496,15 +496,15 @@ public boolean canRunAppAM(Resource amResource) { // maxAMResource Resource maxResource = Resources.clone(getFairShare()); if (maxResource.getMemorySize() == 0) { - maxResource.setMemory( + maxResource.setMemorySize( Math.min(scheduler.getRootQueueMetrics().getAvailableMB(), getMaxShare().getMemorySize())); } - if (maxResource.getVirtualCoresSize() == 0) { + if (maxResource.getVirtualCores() == 0) { maxResource.setVirtualCores(Math.min( scheduler.getRootQueueMetrics().getAvailableVirtualCores(), - getMaxShare().getVirtualCoresSize())); + getMaxShare().getVirtualCores())); } Resource maxAMResource = Resources.multiply(maxResource, maxAMShare); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java index 5f500cd949..440c73cefd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java @@ -266,10 +266,10 @@ private static long getResourceValue(Resource resource, ResourceType type) { private static void setResourceValue(long val, Resource resource, ResourceType type) { switch (type) { case MEMORY: - resource.setMemory(val); + resource.setMemorySize(val); break; case CPU: - resource.setVirtualCores(val); + resource.setVirtualCores((int)val); break; default: throw new IllegalArgumentException("Invalid resource"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 76d43e0c4f..d198bb97af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -28,7 +28,7 @@ @XmlAccessorType(XmlAccessType.FIELD) public class ResourceInfo { long memory; - long vCores; + int vCores; public ResourceInfo() { } @@ -42,7 +42,7 @@ public long getMemorySize() { return memory; } - public long getvCores() { + public int getvCores() { return vCores; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index f2b4ea7a4d..dc17f5f0f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Logger; -import org.junit.Assert; public class MockAM { @@ -207,7 +206,7 @@ public ResourceRequest createResourceReq(String resource, int memory, int priori pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(memory); + capability.setMemorySize(memory); req.setCapability(capability); if (labelExpression != null) { req.setNodeLabelExpression(labelExpression); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 630a8db5aa..22aa0ee732 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -82,19 +82,19 @@ public static List deactivatedNodes(int racks, int nodesPerRack, public static Resource newResource(int mem) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory(mem); + rs.setMemorySize(mem); return rs; } public static Resource newUsedResource(Resource total) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory((int)(Math.random() * total.getMemorySize())); + rs.setMemorySize((int)(Math.random() * total.getMemorySize())); return rs; } public static Resource newAvailResource(Resource total, Resource used) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory(total.getMemorySize() - used.getMemorySize()); + rs.setMemorySize(total.getMemorySize() - used.getMemorySize()); return rs; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index f1f3d51943..84fe7c0ea6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -455,7 +455,7 @@ public RMApp submitApp(int masterMemory, String name, String user, Map acls, String queue, String amLabel) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, name, user, acls, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, @@ -508,7 +508,7 @@ public RMApp submitApp(int masterMemory, String name, String user, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, false, null, 0, null, true, Priority.newInstance(0)); @@ -517,7 +517,7 @@ public RMApp submitApp(int masterMemory, String name, String user, public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, @@ -532,7 +532,7 @@ public RMApp submitApp(int masterMemory, String name, String user, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, @@ -542,7 +542,7 @@ public RMApp submitApp(int masterMemory, String name, String user, public RMApp submitApp(int masterMemory, LogAggregationContext logAggregationContext) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 498112e008..098ba544da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -826,21 +826,21 @@ public void testNodeRegistrationWithMinimumAllocations() throws Exception { resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction()); - capability.setMemory(2048); + capability.setMemorySize(2048); capability.setVirtualCores(1); req.setResource(capability); RegisterNodeManagerResponse response2 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction()); - capability.setMemory(1024); + capability.setMemorySize(1024); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response3 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction()); - capability.setMemory(2048); + capability.setMemorySize(2048); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response4 = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 3a60e0234f..a9f1f634be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -1345,7 +1345,7 @@ public void testContainerCompleteMsgNotLostAfterAMFailedAndRMRestart() throws Ex // submit app with keepContainersAcrossApplicationAttempts true Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation .getCurrentUser().getShortUserName(), null, false, null, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java index 4cb235fa32..d383bac218 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java @@ -127,7 +127,7 @@ private ReservationAllocation createReservationAllocation(long startTime, public Resource createResource(int memory, int vCores) { Resource resource = new ResourcePBImpl(); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 62b1082f53..cb1104bffb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -402,9 +402,9 @@ public static void checkApps(MetricsSource source, int submitted, int pending, } public static void checkResources(MetricsSource source, long allocatedMB, - long allocatedCores, int allocCtnrs, long aggreAllocCtnrs, - long aggreReleasedCtnrs, long availableMB, long availableCores, long pendingMB, - long pendingCores, int pendingCtnrs, long reservedMB, long reservedCores, + int allocatedCores, int allocCtnrs, long aggreAllocCtnrs, + long aggreReleasedCtnrs, long availableMB, int availableCores, long pendingMB, + int pendingCores, int pendingCtnrs, long reservedMB, int reservedCores, int reservedCtnrs) { MetricsRecordBuilder rb = getMetrics(source); assertGauge("AllocatedMB", allocatedMB, rb); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 74fe534d2f..4aee3c1ec9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -1143,7 +1143,7 @@ public void testAppSubmissionWithPreviousToken() throws Exception{ // submit app1 with a token, set cancelTokenWhenComplete to false; Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, false, null); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); @@ -1205,7 +1205,7 @@ public void testCancelWithMultipleAppSubmissions() throws Exception{ Assert.assertFalse(Renewer.cancelled); Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true, null);