big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-890. Ensure CapacityScheduler doesn't round-up metric for available resources. Contributed by Xuan Gong & Hitesh Shah.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1529015 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2013-10-03 21:54:35 +00:00
parent fb3f338c65
commit 79a11ce09d
3 changed files with 17 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -134,6 +134,9 @@ Release 2.1.2 - UNRELEASED
YARN-876. Node resource is added twice when node comes back from unhealthy YARN-876. Node resource is added twice when node comes back from unhealthy
to healthy. (Peng Zhang via Sandy Ryza) to healthy. (Peng Zhang via Sandy Ryza)
YARN-890. Ensure CapacityScheduler doesn't round-up metric for available
resources. (Xuan Gong & Hitesh Shah via acmurthy)
Release 2.1.1-beta - 2013-09-23 Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
View File

@ -99,15 +99,11 @@ public static void updateQueueStatistics(
Resources.divide(calculator, clusterResource, Resources.divide(calculator, clusterResource,
usedResources, queueLimit); usedResources, queueLimit);
} }
childQueue.setUsedCapacity(usedCapacity); childQueue.setUsedCapacity(usedCapacity);
childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity); childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
Resource available = Resource available = Resources.subtract(queueLimit, usedResources);
Resources.roundUp(
calculator,
Resources.subtract(queueLimit, usedResources),
minimumAllocation);
childQueue.getMetrics().setAvailableResourcesToQueue( childQueue.getMetrics().setAvailableResourcesToQueue(
Resources.max( Resources.max(
calculator, calculator,

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
View File

@ -283,8 +283,9 @@ public void testSingleQueueOneUserMetrics() throws Exception {
// Setup some nodes // Setup some nodes
String host_0 = "127.0.0.1"; String host_0 = "127.0.0.1";
FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 8*GB); FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
8*GB);
final int numNodes = 1; final int numNodes = 1;
Resource clusterResource = Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16); Resources.createResource(numNodes * (8*GB), numNodes * 16);
@ -300,7 +301,9 @@ public void testSingleQueueOneUserMetrics() throws Exception {
// Only 1 container // Only 1 container
a.assignContainers(clusterResource, node_0); a.assignContainers(clusterResource, node_0);
assertEquals(6*GB, a.getMetrics().getAvailableMB()); assertEquals(
(int)(node_0.getTotalResource().getMemory() * a.getCapacity()) - (1*GB),
a.getMetrics().getAvailableMB());
} }
@Test @Test
@ -405,8 +408,9 @@ public void testSingleQueueWithOneUser() throws Exception {
// Setup some nodes // Setup some nodes
String host_0 = "127.0.0.1"; String host_0 = "127.0.0.1";
FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 8*GB); FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
8*GB);
final int numNodes = 1; final int numNodes = 1;
Resource clusterResource = Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16); Resources.createResource(numNodes * (8*GB), numNodes * 16);
@ -493,12 +497,14 @@ public void testSingleQueueWithOneUser() throws Exception {
a.completedContainer(clusterResource, app_1, node_0, rmContainer, a.completedContainer(clusterResource, app_1, node_0, rmContainer,
null, RMContainerEventType.KILL, null); null, RMContainerEventType.KILL, null);
} }
assertEquals(0*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, a.getMetrics().getReservedMB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(0*GB, a.getMetrics().getAllocatedMB()); assertEquals(0*GB, a.getMetrics().getAllocatedMB());
assertEquals(1*GB, a.getMetrics().getAvailableMB()); assertEquals((int)(a.getCapacity() * node_0.getTotalResource().getMemory()),
a.getMetrics().getAvailableMB());
} }
@Test @Test