big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-336. Fair scheduler FIFO scheduling within a queue only allows 1 app at a time. Contributed by Sandy Ryza.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1433526 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas White 2013-01-15 17:09:48 +00:00
parent 8447557333
commit 72e631098d
5 changed files with 52 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -192,6 +192,9 @@ Release 2.0.3-alpha - Unreleased
YARN-335. Fair scheduler doesn't check whether rack needs containers
before assigning to node. (Sandy Ryza via tomwhite)
YARN-336. Fair scheduler FIFO scheduling within a queue only allows 1
app at a time. (Sandy Ryza via tomwhite)
Release 2.0.2-alpha - 2012-09-07

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
View File

@ -275,7 +275,7 @@ private Resource assignContainer(FSSchedulerNode node,
// The desired container won't fit here, so reserve
reserve(application, priority, node, container, reserved);
return Resources.none();
return FairScheduler.CONTAINER_RESERVED;
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
View File

@ -177,7 +177,10 @@ public Resource assignContainer(FSSchedulerNode node, boolean reserved) {
Collections.sort(appScheds, comparator);
for (AppSchedulable sched: appScheds) {
if (sched.getRunnable()) {
return sched.assignContainer(node, reserved);
Resource assignedResource = sched.assignContainer(node, reserved);
if (!assignedResource.equals(Resources.none())) {
return assignedResource;
}
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
View File

@ -109,6 +109,10 @@ public class FairScheduler implements ResourceScheduler {
private Clock clock;
private static final Log LOG = LogFactory.getLog(FairScheduler.class);
// Value that container assignment methods return when a container is
// reserved
public static final Resource CONTAINER_RESERVED = Resources.createResource(-1);
// How often fair shares are re-calculated (ms)
protected long UPDATE_INTERVAL = 500;

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
View File

@ -1317,4 +1317,44 @@ public void testMultipleNodesSingleRackRequest() throws Exception {
// should assign rack local
assertEquals(2, scheduler.applications.get(appId).getLiveContainers().size());
}
@Test
public void testFifoWithinQueue() throws Exception {
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(3072));
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
// Even if submitted at exact same time, apps will be deterministically
// ordered by name.
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1",
"user1", 2);
ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1",
"user1", 2);
FSSchedulerApp app1 = scheduler.applications.get(attId1);
FSSchedulerApp app2 = scheduler.applications.get(attId2);
FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1");
queue1.setSchedulingMode(SchedulingMode.FIFO);
scheduler.update();
// First two containers should go to app 1, third should go to app 2.
// Because tests set assignmultiple to false, each heartbeat assigns a single
// container.
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1,
new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
scheduler.handle(updateEvent);
assertEquals(1, app1.getLiveContainers().size());
assertEquals(0, app2.getLiveContainers().size());
scheduler.handle(updateEvent);
assertEquals(2, app1.getLiveContainers().size());
assertEquals(0, app2.getLiveContainers().size());
scheduler.handle(updateEvent);
assertEquals(2, app1.getLiveContainers().size());
assertEquals(1, app2.getLiveContainers().size());
}
}