From 723f1f61d681c137cb2ba224b80a1391bac48aa9 Mon Sep 17 00:00:00 2001 From: Eric Badger Date: Wed, 5 Aug 2020 18:56:56 +0000 Subject: [PATCH] YARN-4575. ApplicationResourceUsageReport should return ALL reserved resource. Contributed by Bibin Chundatt and Eric Payne. (cherry picked from commit 5edd8b925ef22b83350a21abed6ecc551adb92ee) --- .../SchedulerApplicationAttempt.java | 4 +- .../common/fica/FiCaSchedulerApp.java | 3 +- .../TestSchedulerApplicationAttempt.java | 38 +++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 657c03cfd1..4a8f670b4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -1126,8 +1126,8 @@ public ApplicationResourceUsageReport getResourceUsageReport() { getRunningAggregateAppResourceUsage(); Resource usedResourceClone = Resources.clone( attemptResourceUsage.getAllUsed()); - Resource reservedResourceClone = Resources.clone( - attemptResourceUsage.getReserved()); + Resource reservedResourceClone = + Resources.clone(attemptResourceUsage.getAllReserved()); Resource cluster = rmContext.getScheduler().getClusterResource(); ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index cf6ffd9823..6542d1ade6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -736,7 +736,8 @@ private boolean internalUnreserve(FiCaSchedulerNode node, + " on node " + node + ", currently has " + reservedContainers.size() + " at priority " + schedulerKey.getPriority() - + "; currentReservation " + this.attemptResourceUsage.getReserved() + + "; currentReservation " + + this.attemptResourceUsage.getReserved(node.getPartition()) + " on node-label=" + node.getPartition()); return true; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java index 66698201c2..b1080f7d5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -320,6 +321,43 @@ public void testAppPercentagesOnswitch() throws Exception { 0.0f); } + @Test + public void testAllResourceUsage() throws Exception { + FifoScheduler scheduler = mock(FifoScheduler.class); + when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(0, 0)); + when(scheduler.getResourceCalculator()) + .thenReturn(new DefaultResourceCalculator()); + + ApplicationAttemptId appAttId = createAppAttemptId(0, 0); + RMContext rmContext = mock(RMContext.class); + when(rmContext.getEpoch()).thenReturn(3L); + when(rmContext.getScheduler()).thenReturn(scheduler); + when(rmContext.getYarnConfiguration()).thenReturn(conf); + + final String user = "user1"; + Queue queue = createQueue("test", null); + SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, + user, queue, queue.getAbstractUsersManager(), rmContext); + + // Resource request + Resource requestedResource = Resource.newInstance(1536, 2); + app.attemptResourceUsage.incUsed("X", requestedResource); + app.attemptResourceUsage.incUsed("Y", requestedResource); + Resource r2 = Resource.newInstance(1024, 1); + app.attemptResourceUsage.incReserved("X", r2); + app.attemptResourceUsage.incReserved("Y", r2); + + assertTrue("getUsedResources expected " + Resource.newInstance(3072, 4) + + " but was " + app.getResourceUsageReport().getUsedResources(), + Resources.equals(Resource.newInstance(3072, 4), + app.getResourceUsageReport().getUsedResources())); + assertTrue("getReservedResources expected " + Resource.newInstance(2048, 2) + + " but was " + + app.getResourceUsageReport().getReservedResources(), + Resources.equals(Resource.newInstance(2048, 2), + app.getResourceUsageReport().getReservedResources())); + } + @Test public void testSchedulingOpportunityOverflow() throws Exception { ApplicationAttemptId attemptId = createAppAttemptId(0, 0);