big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4575. ApplicationResourceUsageReport should return ALL reserved resource.

Browse Source 
Contributed by Bibin Chundatt and Eric Payne.

(cherry picked from commit 5edd8b925e)
This commit is contained in:
Eric Badger 2020-08-05 18:56:56 +00:00
parent 1cd1f978da
commit 723f1f61d6
3 changed files with 42 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
View File

@ -1126,8 +1126,8 @@ public ApplicationResourceUsageReport getResourceUsageReport() {
getRunningAggregateAppResourceUsage();
Resource usedResourceClone = Resources.clone(
attemptResourceUsage.getAllUsed());
Resource reservedResourceClone = Resources.clone(
attemptResourceUsage.getReserved());
Resource reservedResourceClone =
Resources.clone(attemptResourceUsage.getAllReserved());
Resource cluster = rmContext.getScheduler().getClusterResource();
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
View File

@ -736,7 +736,8 @@ private boolean internalUnreserve(FiCaSchedulerNode node,
+ " on node " + node + ", currently has "
+ reservedContainers.size()
+ " at priority " + schedulerKey.getPriority()
+ "; currentReservation " + this.attemptResourceUsage.getReserved()
+ "; currentReservation "
+ this.attemptResourceUsage.getReserved(node.getPartition())
+ " on node-label=" + node.getPartition());
return true;
}

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
View File

@ -47,6 +47,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@ -320,6 +321,43 @@ public void testAppPercentagesOnswitch() throws Exception {
0.0f);
}
@Test
public void testAllResourceUsage() throws Exception {
FifoScheduler scheduler = mock(FifoScheduler.class);
when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(0, 0));
when(scheduler.getResourceCalculator())
.thenReturn(new DefaultResourceCalculator());
ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
RMContext rmContext = mock(RMContext.class);
when(rmContext.getEpoch()).thenReturn(3L);
when(rmContext.getScheduler()).thenReturn(scheduler);
when(rmContext.getYarnConfiguration()).thenReturn(conf);
final String user = "user1";
Queue queue = createQueue("test", null);
SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId,
user, queue, queue.getAbstractUsersManager(), rmContext);
// Resource request
Resource requestedResource = Resource.newInstance(1536, 2);
app.attemptResourceUsage.incUsed("X", requestedResource);
app.attemptResourceUsage.incUsed("Y", requestedResource);
Resource r2 = Resource.newInstance(1024, 1);
app.attemptResourceUsage.incReserved("X", r2);
app.attemptResourceUsage.incReserved("Y", r2);
assertTrue("getUsedResources expected " + Resource.newInstance(3072, 4)
+ " but was " + app.getResourceUsageReport().getUsedResources(),
Resources.equals(Resource.newInstance(3072, 4),
app.getResourceUsageReport().getUsedResources()));
assertTrue("getReservedResources expected " + Resource.newInstance(2048, 2)
+ " but was "
+ app.getResourceUsageReport().getReservedResources(),
Resources.equals(Resource.newInstance(2048, 2),
app.getResourceUsageReport().getReservedResources()));
}
@Test
public void testSchedulingOpportunityOverflow() throws Exception {
ApplicationAttemptId attemptId = createAppAttemptId(0, 0);