YARN-23. FairScheduler: FSQueueSchedulable#updateDemand() - potential redundant aggregation. (kkambatl via tucu)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1394321 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
016b86867b
commit
b8f81d74c8
@ -43,6 +43,9 @@ Release 2.0.3-alpha - Unreleased
|
|||||||
YARN-116. Add the ability to change the RM include/exclude file without
|
YARN-116. Add the ability to change the RM include/exclude file without
|
||||||
a restart. (xieguiming and Harsh J via sseth)
|
a restart. (xieguiming and Harsh J via sseth)
|
||||||
|
|
||||||
|
YARN-23. FairScheduler: FSQueueSchedulable#updateDemand() - potential
|
||||||
|
redundant aggregation. (kkambatl via tucu)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
BUG FIXES
|
BUG FIXES
|
||||||
|
@ -65,6 +65,17 @@ public class FSQueueSchedulable extends Schedulable implements Queue {
|
|||||||
long lastTimeAtMinShare;
|
long lastTimeAtMinShare;
|
||||||
long lastTimeAtHalfFairShare;
|
long lastTimeAtHalfFairShare;
|
||||||
|
|
||||||
|
// Constructor for tests
|
||||||
|
protected FSQueueSchedulable(FairScheduler scheduler, FSQueue fsQueue,
|
||||||
|
QueueManager qMgr, QueueMetrics metrics, long minShare, long fairShare) {
|
||||||
|
this.scheduler = scheduler;
|
||||||
|
this.queueMgr = qMgr;
|
||||||
|
this.queue = fsQueue;
|
||||||
|
this.metrics = metrics;
|
||||||
|
this.lastTimeAtMinShare = minShare;
|
||||||
|
this.lastTimeAtHalfFairShare = fairShare;
|
||||||
|
}
|
||||||
|
|
||||||
public FSQueueSchedulable(FairScheduler scheduler, FSQueue queue) {
|
public FSQueueSchedulable(FairScheduler scheduler, FSQueue queue) {
|
||||||
this.scheduler = scheduler;
|
this.scheduler = scheduler;
|
||||||
this.queue = queue;
|
this.queue = queue;
|
||||||
@ -93,19 +104,27 @@ public void removeApp(FSSchedulerApp app) {
|
|||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void updateDemand() {
|
public void updateDemand() {
|
||||||
|
// Compute demand by iterating through apps in the queue
|
||||||
|
// Limit demand to maxResources
|
||||||
|
Resource maxRes = queueMgr.getMaxResources(queue.getName());
|
||||||
demand = Resources.createResource(0);
|
demand = Resources.createResource(0);
|
||||||
for (AppSchedulable sched: appScheds) {
|
for (AppSchedulable sched: appScheds) {
|
||||||
sched.updateDemand();
|
sched.updateDemand();
|
||||||
Resource toAdd = sched.getDemand();
|
Resource toAdd = sched.getDemand();
|
||||||
LOG.debug("Counting resource from " + sched.getName() + " " + toAdd.toString());
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("Total resource consumption for " + this.getName() + " now " + demand.toString());
|
LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
|
||||||
|
+ "; Total resource consumption for " + this.getName() + " now "
|
||||||
|
+ demand);
|
||||||
|
}
|
||||||
demand = Resources.add(demand, toAdd);
|
demand = Resources.add(demand, toAdd);
|
||||||
|
if (Resources.greaterThanOrEqual(demand, maxRes)) {
|
||||||
|
demand = maxRes;
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// if demand exceeds the cap for this queue, limit to the max
|
if (LOG.isDebugEnabled()) {
|
||||||
Resource maxRes = queueMgr.getMaxResources(queue.getName());
|
LOG.debug("The updated demand for " + this.getName() + " is " + demand
|
||||||
if(Resources.greaterThan(demand, maxRes)) {
|
+ "; the max is " + maxRes);
|
||||||
demand = maxRes;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -0,0 +1,42 @@
|
|||||||
|
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
public class TestFSQueueSchedulable {
|
||||||
|
private FSQueueSchedulable schedulable = null;
|
||||||
|
private Resource maxResource = Resources.createResource(10);
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setup() {
|
||||||
|
String queueName = "testFSQueue";
|
||||||
|
FSQueue mockQueue = mock(FSQueue.class);
|
||||||
|
when(mockQueue.getName()).thenReturn(queueName);
|
||||||
|
|
||||||
|
QueueManager mockMgr = mock(QueueManager.class);
|
||||||
|
when(mockMgr.getMaxResources(queueName)).thenReturn(maxResource);
|
||||||
|
|
||||||
|
schedulable = new FSQueueSchedulable(null, mockQueue, mockMgr, null, 0, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUpdateDemand() {
|
||||||
|
AppSchedulable app = mock(AppSchedulable.class);
|
||||||
|
Mockito.when(app.getDemand()).thenReturn(maxResource);
|
||||||
|
|
||||||
|
schedulable.addApp(app);
|
||||||
|
schedulable.addApp(app);
|
||||||
|
|
||||||
|
schedulable.updateDemand();
|
||||||
|
|
||||||
|
assertTrue("Demand is greater than max allowed ",
|
||||||
|
Resources.equals(schedulable.getDemand(), maxResource));
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user