diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c0c75d072d..799ce495ec 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -538,6 +538,8 @@ Release 0.23.1 - Unreleased MAPREDUCE-3646. Remove redundant URL info from "mapred job" output. (Jonathan Eagles via mahadev) + MAPREDUCE-3681. Fixed computation of queue's usedCapacity. (acmurthy) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 5f1314113e..b8d89878ec 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -272,9 +272,9 @@ public class LeafQueue implements CSQueue { "maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser + " [= (int)(maxActiveApplications * (userLimit / 100.0f) * userLimitFactor) ]" + "\n" + "utilization = " + utilization + - " [= usedResourcesMemory / queueLimit ]" + "\n" + + " [= usedResourcesMemory / (clusterResourceMemory * absoluteCapacity)]" + "\n" + "usedCapacity = " + usedCapacity + - " [= usedResourcesMemory / (clusterResourceMemory * capacity) ]" + "\n" + + " [= usedResourcesMemory / (clusterResourceMemory * parent.absoluteCapacity)]" + "\n" + "maxAMResourcePercent = " + maxAMResourcePercent + " [= configuredMaximumAMResourcePercent ]" + "\n" + "minimumAllocationFactor = " + minimumAllocationFactor + @@ -502,9 +502,14 @@ public class LeafQueue implements CSQueue { } public String toString() { - return queueName + ":" + capacity + ":" + absoluteCapacity + ":" + - getUsedCapacity() + ":" + getUtilization() + ":" + - getNumApplications() + ":" + getNumContainers(); + return queueName + ": " + + "capacity=" + capacity + ", " + + "absoluteCapacity=" + absoluteCapacity + ", " + + "usedResources=" + usedResources.getMemory() + "MB, " + + "usedCapacity=" + getUsedCapacity() + ", " + + "utilization=" + getUtilization() + ", " + + "numApps=" + getNumApplications() + ", " + + "numContainers=" + getNumContainers(); } private synchronized User getUser(String userName) { @@ -1316,11 +1321,11 @@ public class LeafQueue implements CSQueue { } private synchronized void updateResource(Resource clusterResource) { - float queueLimit = clusterResource.getMemory() * absoluteCapacity; + float queueLimit = clusterResource.getMemory() * absoluteCapacity; setUtilization(usedResources.getMemory() / queueLimit); - setUsedCapacity( - usedResources.getMemory() / (clusterResource.getMemory() * capacity)); - + setUsedCapacity(usedResources.getMemory() + / (clusterResource.getMemory() * parent.getAbsoluteCapacity())); + Resource resourceLimit = Resources.createResource(roundUp((int)queueLimit)); metrics.setAvailableResourcesToQueue( diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 4fea3227c1..41ef854847 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -333,10 +333,15 @@ public class ParentQueue implements CSQueue { } public String toString() { - return queueName + ":" + capacity + ":" + absoluteCapacity + ":" + - getUsedCapacity() + ":" + getUtilization() + ":" + - getNumApplications() + ":" + getNumContainers() + ":" + - childQueues.size() + " child-queues"; + return queueName + ": " + + "numChildQueue= " + childQueues.size() + ", " + + "capacity=" + capacity + ", " + + "absoluteCapacity=" + absoluteCapacity + ", " + + "usedResources=" + usedResources.getMemory() + "MB, " + + "usedCapacity=" + getUsedCapacity() + ", " + + "utilization=" + getUtilization() + ", " + + "numApps=" + getNumApplications() + ", " + + "numContainers=" + getNumContainers(); } @Override @@ -688,11 +693,13 @@ public class ParentQueue implements CSQueue { } private synchronized void updateResource(Resource clusterResource) { - float queueLimit = clusterResource.getMemory() * absoluteCapacity; + float queueLimit = clusterResource.getMemory() * absoluteCapacity; + float parentAbsoluteCapacity = + (rootQueue) ? 1.0f : parent.getAbsoluteCapacity(); setUtilization(usedResources.getMemory() / queueLimit); - setUsedCapacity( - usedResources.getMemory() / (clusterResource.getMemory() * capacity)); - + setUsedCapacity(usedResources.getMemory() + / (clusterResource.getMemory() * parentAbsoluteCapacity)); + Resource resourceLimit = Resources.createResource((int)queueLimit); metrics.setAvailableResourcesToQueue( diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index bbfd503b4d..6bddf87eeb 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -138,12 +138,34 @@ public class TestParentQueue { when(queue).assignContainers(eq(clusterResource), eq(node)); } + private float computeQueueUsedCapacity(CSQueue queue, + int expectedMemory, Resource clusterResource) { + return ( + ((float)expectedMemory / clusterResource.getMemory()) * + queue.getParent().getAbsoluteCapacity() + ); + } + private float computeQueueUtilization(CSQueue queue, int expectedMemory, Resource clusterResource) { return (expectedMemory / (clusterResource.getMemory() * queue.getAbsoluteCapacity())); } + final static float DELTA = 0.0001f; + private void verifyQueueMetrics(CSQueue queue, + int expectedMemory, Resource clusterResource) { + assertEquals( + computeQueueUtilization(queue, expectedMemory, clusterResource), + queue.getUtilization(), + DELTA); + assertEquals( + computeQueueUsedCapacity(queue, expectedMemory, clusterResource), + queue.getUsedCapacity(), + DELTA); + + } + @Test public void testSingleLevelQueues() throws Exception { // Setup queue configs @@ -173,15 +195,13 @@ public class TestParentQueue { // Start testing LeafQueue a = (LeafQueue)queues.get(A); LeafQueue b = (LeafQueue)queues.get(B); - final float delta = 0.0001f; // Simulate B returning a container on node_0 stubQueueAllocation(a, clusterResource, node_0, 0*GB); stubQueueAllocation(b, clusterResource, node_0, 1*GB); root.assignContainers(clusterResource, node_0); - assertEquals(0.0f, a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 1*GB, clusterResource), - b.getUtilization(), delta); + verifyQueueMetrics(a, 0*GB, clusterResource); + verifyQueueMetrics(b, 1*GB, clusterResource); // Now, A should get the scheduling opportunity since A=0G/6G, B=1G/14G stubQueueAllocation(a, clusterResource, node_1, 2*GB); @@ -192,10 +212,8 @@ public class TestParentQueue { any(SchedulerNode.class)); allocationOrder.verify(b).assignContainers(eq(clusterResource), any(SchedulerNode.class)); - assertEquals(computeQueueUtilization(a, 2*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 2*GB, clusterResource), - b.getUtilization(), delta); + verifyQueueMetrics(a, 2*GB, clusterResource); + verifyQueueMetrics(b, 2*GB, clusterResource); // Now, B should get the scheduling opportunity // since A has 2/6G while B has 2/14G @@ -207,10 +225,8 @@ public class TestParentQueue { any(SchedulerNode.class)); allocationOrder.verify(a).assignContainers(eq(clusterResource), any(SchedulerNode.class)); - assertEquals(computeQueueUtilization(a, 3*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 4*GB, clusterResource), - b.getUtilization(), delta); + verifyQueueMetrics(a, 3*GB, clusterResource); + verifyQueueMetrics(b, 4*GB, clusterResource); // Now, B should still get the scheduling opportunity // since A has 3/6G while B has 4/14G @@ -222,10 +238,8 @@ public class TestParentQueue { any(SchedulerNode.class)); allocationOrder.verify(a).assignContainers(eq(clusterResource), any(SchedulerNode.class)); - assertEquals(computeQueueUtilization(a, 3*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 8*GB, clusterResource), - b.getUtilization(), delta); + verifyQueueMetrics(a, 3*GB, clusterResource); + verifyQueueMetrics(b, 8*GB, clusterResource); // Now, A should get the scheduling opportunity // since A has 3/6G while B has 8/14G @@ -237,10 +251,8 @@ public class TestParentQueue { any(SchedulerNode.class)); allocationOrder.verify(a).assignContainers(eq(clusterResource), any(SchedulerNode.class)); - assertEquals(computeQueueUtilization(a, 4*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 9*GB, clusterResource), - b.getUtilization(), delta); + verifyQueueMetrics(a, 4*GB, clusterResource); + verifyQueueMetrics(b, 9*GB, clusterResource); } private static final String C = "c"; @@ -323,22 +335,16 @@ public class TestParentQueue { CSQueue b2 = queues.get(B2); CSQueue b3 = queues.get(B3); - final float delta = 0.0001f; - // Simulate C returning a container on node_0 stubQueueAllocation(a, clusterResource, node_0, 0*GB); stubQueueAllocation(b, clusterResource, node_0, 0*GB); stubQueueAllocation(c, clusterResource, node_0, 1*GB); stubQueueAllocation(d, clusterResource, node_0, 0*GB); root.assignContainers(clusterResource, node_0); - assertEquals(computeQueueUtilization(a, 0*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 0*GB, clusterResource), - b.getUtilization(), delta); - assertEquals(computeQueueUtilization(c, 1*GB, clusterResource), - c.getUtilization(), delta); - assertEquals(computeQueueUtilization(d, 0*GB, clusterResource), - d.getUtilization(), delta); + verifyQueueMetrics(a, 0*GB, clusterResource); + verifyQueueMetrics(b, 0*GB, clusterResource); + verifyQueueMetrics(c, 1*GB, clusterResource); + verifyQueueMetrics(d, 0*GB, clusterResource); reset(a); reset(b); reset(c); // Now get B2 to allocate @@ -347,12 +353,9 @@ public class TestParentQueue { stubQueueAllocation(b2, clusterResource, node_1, 4*GB); stubQueueAllocation(c, clusterResource, node_1, 0*GB); root.assignContainers(clusterResource, node_1); - assertEquals(computeQueueUtilization(a, 0*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 4*GB, clusterResource), - b.getUtilization(), delta); - assertEquals(computeQueueUtilization(c, 1*GB, clusterResource), - c.getUtilization(), delta); + verifyQueueMetrics(a, 0*GB, clusterResource); + verifyQueueMetrics(b, 4*GB, clusterResource); + verifyQueueMetrics(c, 1*GB, clusterResource); reset(a); reset(b); reset(c); // Now get both A1, C & B3 to allocate in right order @@ -368,12 +371,9 @@ public class TestParentQueue { any(SchedulerNode.class)); allocationOrder.verify(b).assignContainers(eq(clusterResource), any(SchedulerNode.class)); - assertEquals(computeQueueUtilization(a, 1*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 6*GB, clusterResource), - b.getUtilization(), delta); - assertEquals(computeQueueUtilization(c, 3*GB, clusterResource), - c.getUtilization(), delta); + verifyQueueMetrics(a, 1*GB, clusterResource); + verifyQueueMetrics(b, 6*GB, clusterResource); + verifyQueueMetrics(c, 3*GB, clusterResource); reset(a); reset(b); reset(c); // Now verify max-capacity @@ -399,16 +399,12 @@ public class TestParentQueue { any(SchedulerNode.class)); allocationOrder.verify(c).assignContainers(eq(clusterResource), any(SchedulerNode.class)); - assertEquals(computeQueueUtilization(a, 3*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 8*GB, clusterResource), - b.getUtilization(), delta); - assertEquals(computeQueueUtilization(c, 4*GB, clusterResource), - c.getUtilization(), delta); + verifyQueueMetrics(a, 3*GB, clusterResource); + verifyQueueMetrics(b, 8*GB, clusterResource); + verifyQueueMetrics(c, 4*GB, clusterResource); reset(a); reset(b); reset(c); - } - + @Test public void testOffSwitchScheduling() throws Exception { // Setup queue configs @@ -438,15 +434,13 @@ public class TestParentQueue { // Start testing LeafQueue a = (LeafQueue)queues.get(A); LeafQueue b = (LeafQueue)queues.get(B); - final float delta = 0.0001f; // Simulate B returning a container on node_0 stubQueueAllocation(a, clusterResource, node_0, 0*GB, NodeType.OFF_SWITCH); stubQueueAllocation(b, clusterResource, node_0, 1*GB, NodeType.OFF_SWITCH); root.assignContainers(clusterResource, node_0); - assertEquals(0.0f, a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 1*GB, clusterResource), - b.getUtilization(), delta); + verifyQueueMetrics(a, 0*GB, clusterResource); + verifyQueueMetrics(b, 1*GB, clusterResource); // Now, A should get the scheduling opportunity since A=0G/6G, B=1G/14G // also, B gets a scheduling opportunity since A allocates RACK_LOCAL @@ -458,10 +452,8 @@ public class TestParentQueue { any(SchedulerNode.class)); allocationOrder.verify(b).assignContainers(eq(clusterResource), any(SchedulerNode.class)); - assertEquals(computeQueueUtilization(a, 2*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 2*GB, clusterResource), - b.getUtilization(), delta); + verifyQueueMetrics(a, 2*GB, clusterResource); + verifyQueueMetrics(b, 2*GB, clusterResource); // Now, B should get the scheduling opportunity // since A has 2/6G while B has 2/14G, @@ -474,10 +466,8 @@ public class TestParentQueue { any(SchedulerNode.class)); allocationOrder.verify(a).assignContainers(eq(clusterResource), any(SchedulerNode.class)); - assertEquals(computeQueueUtilization(a, 2*GB, clusterResource), - a.getUtilization(), delta); - assertEquals(computeQueueUtilization(b, 4*GB, clusterResource), - b.getUtilization(), delta); + verifyQueueMetrics(a, 2*GB, clusterResource); + verifyQueueMetrics(b, 4*GB, clusterResource); }