From 224fc101fdfa355b4ad842fc39192d8e4f1f2979 Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Fri, 9 Dec 2011 01:38:35 +0000 Subject: [PATCH] MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value for queues (Anupam Seth via mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1212212 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../scheduler/capacity/CSQueueUtils.java | 2 +- .../CapacitySchedulerConfiguration.java | 2 ++ .../scheduler/capacity/LeafQueue.java | 20 ++++++------- .../scheduler/capacity/ParentQueue.java | 10 +++---- .../scheduler/capacity/TestLeafQueue.java | 30 ++++++++++++++++++- 6 files changed, 50 insertions(+), 17 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e316a6bc21..c73df6b572 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -252,6 +252,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3147. Handle leaf queues with the same name properly. (Ravi Prakash via mahadev) + MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value + for queues (Anupam Seth via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 13491208dd..c238a8ad95 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -21,7 +21,7 @@ class CSQueueUtils { public static void checkMaxCapacity(String queueName, float capacity, float maximumCapacity) { - if (maximumCapacity != CapacitySchedulerConfiguration.UNDEFINED && + if (Math.round(100 * maximumCapacity) != CapacitySchedulerConfiguration.UNDEFINED && maximumCapacity < capacity) { throw new IllegalArgumentException( "Illegal call to setMaxCapacity. " + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index dc28c3cb43..cf52a72b67 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -162,6 +162,8 @@ public int getMaximumCapacity(String queue) { public void setMaximumCapacity(String queue, int maxCapacity) { setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); + LOG.info("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) + + ", maxCapacity=" + maxCapacity); } public int getUserLimit(String queue) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index c97aeb79a6..37bfd8cc0f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -146,10 +146,10 @@ public LeafQueue(CapacitySchedulerContext cs, (float)cs.getConfiguration().getCapacity(getQueuePath()) / 100; float absoluteCapacity = parent.getAbsoluteCapacity() * capacity; - float maximumCapacity = cs.getConfiguration().getMaximumCapacity(getQueuePath()); + float maximumCapacity = (float)cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100; float absoluteMaxCapacity = - (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? - Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity) / 100; + (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ? + Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity); int userLimit = cs.getConfiguration().getUserLimit(getQueuePath()); float userLimitFactor = @@ -402,7 +402,7 @@ synchronized void setMaxCapacity(float maximumCapacity) { this.maximumCapacity = maximumCapacity; this.absoluteMaxCapacity = - (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? + (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ? Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity); } @@ -829,13 +829,13 @@ private synchronized boolean assignToQueue(Resource clusterResource, float potentialNewCapacity = (float)(usedResources.getMemory() + required.getMemory()) / clusterResource.getMemory(); + LOG.info(getQueueName() + + " usedResources: " + usedResources.getMemory() + + " currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() + + " required " + required.getMemory() + + " potentialNewCapacity: " + potentialNewCapacity + " ( " + + " max-capacity: " + absoluteMaxCapacity + ")"); if (potentialNewCapacity > absoluteMaxCapacity) { - LOG.info(getQueueName() + - " usedResources: " + usedResources.getMemory() + - " currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() + - " required " + required.getMemory() + - " potentialNewCapacity: " + potentialNewCapacity + " ( " + - " > max-capacity (" + absoluteMaxCapacity + ")"); return false; } return true; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 38664f0db3..7d912009cf 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -123,10 +123,10 @@ public ParentQueue(CapacitySchedulerContext cs, float absoluteCapacity = parentAbsoluteCapacity * capacity; float maximumCapacity = - cs.getConfiguration().getMaximumCapacity(getQueuePath()); + (float) cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100; float absoluteMaxCapacity = - (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? - 1000000000f : (parentAbsoluteCapacity * maximumCapacity) / 100; + (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ? + Float.MAX_VALUE : (parentAbsoluteCapacity * maximumCapacity); QueueState state = cs.getConfiguration().getState(getQueuePath()); @@ -233,12 +233,12 @@ public synchronized float getAbsoluteCapacity() { @Override public float getAbsoluteMaximumCapacity() { - return 0; + return absoluteMaxCapacity; } @Override public float getMaximumCapacity() { - return 0; + return maximumCapacity; } @Override diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index dba1e143cd..949d952192 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -107,12 +107,15 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B}); conf.setCapacity(CapacityScheduler.ROOT, 100); + conf.setMaximumCapacity(CapacityScheduler.ROOT, 100); final String Q_A = CapacityScheduler.ROOT + "." + A; conf.setCapacity(Q_A, 10); + conf.setMaximumCapacity(Q_A, 20); final String Q_B = CapacityScheduler.ROOT + "." + B; conf.setCapacity(Q_B, 90); + conf.setMaximumCapacity(Q_B, 99); LOG.info("Setup top-level queues a and b"); } @@ -158,6 +161,23 @@ public Container answer(InvocationOnMock invocation) return queue; } + @Test + public void testInitializeQueue() throws Exception { + final float epsilon = 1e-5f; + //can add more sturdy test with 3-layer queues + //once MAPREDUCE:3410 is resolved + LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); + assertEquals(0.1, a.getCapacity(), epsilon); + assertEquals(0.1, a.getAbsoluteCapacity(), epsilon); + assertEquals(0.2, a.getMaximumCapacity(), epsilon); + assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon); + + LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B)); + assertEquals(0.9, b.getCapacity(), epsilon); + assertEquals(0.9, b.getAbsoluteCapacity(), epsilon); + assertEquals(0.99, b.getMaximumCapacity(), epsilon); + assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon); + } @Test public void testSingleQueueOneUserMetrics() throws Exception { @@ -209,6 +229,8 @@ public void testSingleQueueWithOneUser() throws Exception { // Manipulate queue 'a' LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); + //unset maxCapacity + a.setMaxCapacity(-0.01f); // Users final String user_0 = "user_0"; @@ -329,6 +351,8 @@ public void testSingleQueueWithMultipleUsers() throws Exception { // Mock the queue LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); + //unset maxCapacity + a.setMaxCapacity(-0.01f); // Users final String user_0 = "user_0"; @@ -442,7 +466,7 @@ public void testSingleQueueWithMultipleUsers() throws Exception { // Revert max-capacity and user-limit-factor // Now, allocations should goto app_3 since it's under user-limit - a.setMaxCapacity(-1); + a.setMaxCapacity(-0.01f); a.setUserLimitFactor(1); a.assignContainers(clusterResource, node_0); assertEquals(7*GB, a.getUsedResources().getMemory()); @@ -498,6 +522,8 @@ public void testReservation() throws Exception { // Manipulate queue 'a' LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); + //unset maxCapacity + a.setMaxCapacity(-0.01f); // Users final String user_0 = "user_0"; @@ -594,6 +620,8 @@ public void testReservationExchange() throws Exception { // Manipulate queue 'a' LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); + //unset maxCapacity + a.setMaxCapacity(-0.01f); a.setUserLimitFactor(10); // Users