MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value for queues (Anupam Seth via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1212212 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -252,6 +252,9 @@ Release 0.23.1 - Unreleased
     MAPREDUCE-3147. Handle leaf queues with the same name properly. (Ravi Prakash via
     mahadev)
 
+    MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value 
+    for queues (Anupam Seth via mahadev)
+
 Release 0.23.0 - 2011-11-01 
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java

@@ -21,7 +21,7 @@ class CSQueueUtils {
   
   public static void checkMaxCapacity(String queueName, 
       float capacity, float maximumCapacity) {
-    if (maximumCapacity != CapacitySchedulerConfiguration.UNDEFINED && 
+    if (Math.round(100 * maximumCapacity) != CapacitySchedulerConfiguration.UNDEFINED && 
         maximumCapacity < capacity) {
       throw new IllegalArgumentException(
           "Illegal call to setMaxCapacity. " +

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java

@@ -162,6 +162,8 @@ public int getMaximumCapacity(String queue) {
   
   public void setMaximumCapacity(String queue, int maxCapacity) {
     setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity);
+    LOG.info("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) + 
+        ", maxCapacity=" + maxCapacity);
   }
   
   public int getUserLimit(String queue) {

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -146,10 +146,10 @@ public LeafQueue(CapacitySchedulerContext cs,
       (float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
     float absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
 
-    float maximumCapacity = cs.getConfiguration().getMaximumCapacity(getQueuePath());
+    float maximumCapacity = (float)cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
     float absoluteMaxCapacity = 
-      (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? 
+      (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ? 
-          Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity) / 100;
+          Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity);
 
     int userLimit = cs.getConfiguration().getUserLimit(getQueuePath());
     float userLimitFactor = 
@@ -402,7 +402,7 @@ synchronized void setMaxCapacity(float maximumCapacity) {
     
     this.maximumCapacity = maximumCapacity;
     this.absoluteMaxCapacity = 
-      (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? 
+      (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ? 
           Float.MAX_VALUE : 
           (parent.getAbsoluteCapacity() * maximumCapacity);
   }
@@ -829,13 +829,13 @@ private synchronized boolean assignToQueue(Resource clusterResource,
     float potentialNewCapacity = 
       (float)(usedResources.getMemory() + required.getMemory()) / 
         clusterResource.getMemory();
+    LOG.info(getQueueName() + 
+        " usedResources: " + usedResources.getMemory() + 
+        " currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() + 
+        " required " + required.getMemory() +
+        " potentialNewCapacity: " + potentialNewCapacity + " ( " +
+        " max-capacity: " + absoluteMaxCapacity + ")");
     if (potentialNewCapacity > absoluteMaxCapacity) {
-      LOG.info(getQueueName() + 
-          " usedResources: " + usedResources.getMemory() + 
-          " currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() + 
-          " required " + required.getMemory() +
-          " potentialNewCapacity: " + potentialNewCapacity + " ( " +
-          " > max-capacity (" + absoluteMaxCapacity + ")");
       return false;
     }
     return true;

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

@@ -123,10 +123,10 @@ public ParentQueue(CapacitySchedulerContext cs,
     float absoluteCapacity = parentAbsoluteCapacity * capacity; 
 
     float maximumCapacity = 
-      cs.getConfiguration().getMaximumCapacity(getQueuePath());
+      (float) cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
     float absoluteMaxCapacity = 
-      (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ? 
+      (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ? 
-          1000000000f :  (parentAbsoluteCapacity * maximumCapacity) / 100;
+          Float.MAX_VALUE :  (parentAbsoluteCapacity * maximumCapacity);
     
     QueueState state = cs.getConfiguration().getState(getQueuePath());
 
@@ -233,12 +233,12 @@ public synchronized float getAbsoluteCapacity() {
 
   @Override
   public float getAbsoluteMaximumCapacity() {
-    return 0;
+    return absoluteMaxCapacity;
   }
 
   @Override
   public float getMaximumCapacity() {
-    return 0;
+    return maximumCapacity;
   }
 
   @Override

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java

@@ -107,12 +107,15 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
     // Define top-level queues
     conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
     conf.setCapacity(CapacityScheduler.ROOT, 100);
+    conf.setMaximumCapacity(CapacityScheduler.ROOT, 100);
     
     final String Q_A = CapacityScheduler.ROOT + "." + A;
     conf.setCapacity(Q_A, 10);
+    conf.setMaximumCapacity(Q_A, 20);
     
     final String Q_B = CapacityScheduler.ROOT + "." + B;
     conf.setCapacity(Q_B, 90);
+    conf.setMaximumCapacity(Q_B, 99);
     
     LOG.info("Setup top-level queues a and b");
   }
@@ -158,6 +161,23 @@ public Container answer(InvocationOnMock invocation)
     return queue;
   }
   
+  @Test
+  public void testInitializeQueue() throws Exception {
+	  final float epsilon = 1e-5f;
+	  //can add more sturdy test with 3-layer queues 
+	  //once MAPREDUCE:3410 is resolved
+	  LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+	  assertEquals(0.1, a.getCapacity(), epsilon);
+	  assertEquals(0.1, a.getAbsoluteCapacity(), epsilon);
+	  assertEquals(0.2, a.getMaximumCapacity(), epsilon);
+	  assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
+	  
+	  LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
+	  assertEquals(0.9, b.getCapacity(), epsilon);
+	  assertEquals(0.9, b.getAbsoluteCapacity(), epsilon);
+	  assertEquals(0.99, b.getMaximumCapacity(), epsilon);
+	  assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
+  }
  
   @Test
   public void testSingleQueueOneUserMetrics() throws Exception {
@@ -209,6 +229,8 @@ public void testSingleQueueWithOneUser() throws Exception {
 
     // Manipulate queue 'a'
     LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+    //unset maxCapacity
+    a.setMaxCapacity(-0.01f);
 
     // Users
     final String user_0 = "user_0";
@@ -329,6 +351,8 @@ public void testSingleQueueWithMultipleUsers() throws Exception {
     
     // Mock the queue
     LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+    //unset maxCapacity
+    a.setMaxCapacity(-0.01f);
     
     // Users
     final String user_0 = "user_0";
@@ -442,7 +466,7 @@ public void testSingleQueueWithMultipleUsers() throws Exception {
     
     // Revert max-capacity and user-limit-factor
     // Now, allocations should goto app_3 since it's under user-limit 
-    a.setMaxCapacity(-1);
+    a.setMaxCapacity(-0.01f);
     a.setUserLimitFactor(1);
     a.assignContainers(clusterResource, node_0);
     assertEquals(7*GB, a.getUsedResources().getMemory()); 
@@ -498,6 +522,8 @@ public void testReservation() throws Exception {
 
     // Manipulate queue 'a'
     LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+    //unset maxCapacity
+    a.setMaxCapacity(-0.01f);
 
     // Users
     final String user_0 = "user_0";
@@ -594,6 +620,8 @@ public void testReservationExchange() throws Exception {
 
     // Manipulate queue 'a'
     LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+    //unset maxCapacity
+    a.setMaxCapacity(-0.01f);
     a.setUserLimitFactor(10);
 
     // Users