From 6d043aa4cf14bc88b1a19991e3bb0752499312b8 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Mon, 23 May 2016 14:29:28 -0700 Subject: [PATCH] YARN-4979. FSAppAttempt demand calculation considers demands at multiple locality levels different. (Zhihai Xu via kasha) --- .../scheduler/fair/FSAppAttempt.java | 3 ++- .../scheduler/fair/TestFairScheduler.java | 17 ++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 482751f4e2..82d665c261 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -929,7 +929,8 @@ public void updateDemand() { // Add up outstanding resource requests synchronized (this) { for (Priority p : getPriorities()) { - for (ResourceRequest r : getResourceRequests(p).values()) { + ResourceRequest r = getResourceRequest(p, ResourceRequest.ANY); + if (r != null) { Resources.multiplyAndAddTo(demand, r.getCapability(), r.getNumContainers()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 5690fa4cdb..d0db94d754 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1691,28 +1691,31 @@ public void testQueueDemandCalculation() throws Exception { ask1.add(request1); scheduler.allocate(id11, ask1, new ArrayList(), null, null, null, null); - // Second ask, queue2 requests 1 large + (2 * minReqSize) + // Second ask, queue2 requests 1 large. List ask2 = new ArrayList(); ResourceRequest request2 = createResourceRequest(2 * minReqSize, "foo", 1, 1, false); - ResourceRequest request3 = createResourceRequest(minReqSize, "bar", 1, 2, - false); + ResourceRequest request3 = createResourceRequest(2 * minReqSize, + ResourceRequest.ANY, 1, 1, false); ask2.add(request2); ask2.add(request3); scheduler.allocate(id21, ask2, new ArrayList(), null, null, null, null); - // Third ask, queue2 requests 1 large + // Third ask, queue2 requests 2 small (minReqSize). List ask3 = new ArrayList(); - ResourceRequest request4 = - createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1, true); + ResourceRequest request4 = createResourceRequest(minReqSize, "bar", 2, 2, + true); + ResourceRequest request5 = createResourceRequest(minReqSize, + ResourceRequest.ANY, 2, 2, true); ask3.add(request4); + ask3.add(request5); scheduler.allocate(id22, ask3, new ArrayList(), null, null, null, null); scheduler.update(); assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("root.queue1") .getDemand().getMemory()); - assertEquals(2 * minReqSize + 2 * minReqSize + (2 * minReqSize), scheduler + assertEquals(2 * minReqSize + 2 * minReqSize, scheduler .getQueueManager().getQueue("root.queue2").getDemand() .getMemory()); }