big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3893. allow capacity scheduler configs max-apps and max-am-pct per queue (tgraves via bobby)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1364764 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-07-23 19:31:33 +00:00
parent b7b8db51de
commit 97ed48e035
5 changed files with 143 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -742,6 +742,10 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't
init. (Jason Lowe via daryn)
MAPREDUCE-3893. allow capacity scheduler configs maximum-applications and
maximum-am-resource-percent configurable on a per queue basis (tgraves via
bobby)
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES

36
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -46,13 +46,21 @@ public class CapacitySchedulerConfiguration extends Configuration {
@Private
public static final String DOT = ".";
@Private
public static final String MAXIMUM_APPLICATIONS_SUFFIX =
"maximum-applications";
@Private
public static final String MAXIMUM_SYSTEM_APPLICATIONS =
PREFIX + "maximum-applications";
PREFIX + MAXIMUM_APPLICATIONS_SUFFIX;
@Private
public static final String MAXIMUM_AM_RESOURCE_SUFFIX =
"maximum-am-resource-percent";
@Private
public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT =
PREFIX + "maximum-am-resource-percent";
PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX;
@Private
public static final String QUEUES = "queues";
@ -131,6 +139,30 @@ public float getMaximumApplicationMasterResourcePercent() {
return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT);
}
/**
* Get the maximum applications per queue setting.
* @param queue name of the queue
* @return setting specified or -1 if not set
*/
public int getMaximumApplicationsPerQueue(String queue) {
int maxApplicationsPerQueue =
getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX,
(int)UNDEFINED);
return maxApplicationsPerQueue;
}
/**
* Get the maximum am resource percent per queue setting.
* @param queue name of the queue
* @return per queue setting or defaults to the global am-resource-percent
* setting if per queue setting not present
*/
public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) {
return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX,
getMaximumApplicationMasterResourcePercent());
}
public float getCapacity(String queue) {
float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED);

37
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -85,7 +85,7 @@ public class LeafQueue implements CSQueue {
private int maxApplications;
private int maxApplicationsPerUser;
private float maxAMResourcePercent;
private float maxAMResourcePerQueuePercent;
private int maxActiveApplications; // Based on absolute max capacity
private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
private int maxActiveApplicationsPerUser;
@ -156,21 +156,25 @@ public LeafQueue(CapacitySchedulerContext cs,
float userLimitFactor =
cs.getConfiguration().getUserLimitFactor(getQueuePath());
int maxSystemJobs = cs.getConfiguration().getMaximumSystemApplications();
int maxApplications = (int)(maxSystemJobs * absoluteCapacity);
int maxApplicationsPerUser =
int maxApplications = cs.getConfiguration().getMaximumApplicationsPerQueue(getQueuePath());
if (maxApplications < 0) {
int maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
maxApplications = (int)(maxSystemApps * absoluteCapacity);
}
maxApplicationsPerUser =
(int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
this.maxAMResourcePercent =
cs.getConfiguration().getMaximumApplicationMasterResourcePercent();
this.maxAMResourcePerQueuePercent =
cs.getConfiguration().
getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
int maxActiveApplications =
CSQueueUtils.computeMaxActiveApplications(
cs.getClusterResources(), this.minimumAllocation,
maxAMResourcePercent, absoluteMaxCapacity);
maxAMResourcePerQueuePercent, absoluteMaxCapacity);
this.maxActiveAppsUsingAbsCap =
CSQueueUtils.computeMaxActiveApplications(
cs.getClusterResources(), this.minimumAllocation,
maxAMResourcePercent, absoluteCapacity);
maxAMResourcePerQueuePercent, absoluteCapacity);
int maxActiveApplicationsPerUser =
CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap, userLimit,
userLimitFactor);
@ -265,15 +269,16 @@ private synchronized void setupQueueConfigs(
"userLimitFactor = " + userLimitFactor +
" [= configuredUserLimitFactor ]" + "\n" +
"maxApplications = " + maxApplications +
" [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]" +
" [= configuredMaximumSystemApplicationsPerQueue or" +
" (int)(configuredMaximumSystemApplications * absoluteCapacity)]" +
"\n" +
"maxApplicationsPerUser = " + maxApplicationsPerUser +
" [= (int)(maxApplications * (userLimit / 100.0f) * " +
"userLimitFactor) ]" + "\n" +
"maxActiveApplications = " + maxActiveApplications +
" [= max(" +
"(int)ceil((clusterResourceMemory / minimumAllocation) *" +
"maxAMResourcePercent * absoluteMaxCapacity)," +
"(int)ceil((clusterResourceMemory / minimumAllocation) * " +
"maxAMResourcePerQueuePercent * absoluteMaxCapacity)," +
"1) ]" + "\n" +
"maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap +
" [= max(" +
@ -290,7 +295,7 @@ private synchronized void setupQueueConfigs(
"(clusterResourceMemory * absoluteCapacity)]" + "\n" +
"absoluteUsedCapacity = " + absoluteUsedCapacity +
" [= usedResourcesMemory / clusterResourceMemory]" + "\n" +
"maxAMResourcePercent = " + maxAMResourcePercent +
"maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent +
" [= configuredMaximumAMResourcePercent ]" + "\n" +
"minimumAllocationFactor = " + minimumAllocationFactor +
" [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " +
@ -1387,11 +1392,11 @@ public synchronized void updateClusterResource(Resource clusterResource) {
maxActiveApplications =
CSQueueUtils.computeMaxActiveApplications(
clusterResource, minimumAllocation,
maxAMResourcePercent, absoluteMaxCapacity);
maxAMResourcePerQueuePercent, absoluteMaxCapacity);
maxActiveAppsUsingAbsCap =
CSQueueUtils.computeMaxActiveApplications(
clusterResource, minimumAllocation,
maxAMResourcePercent, absoluteCapacity);
CSQueueUtils.computeMaxActiveApplications(
clusterResource, minimumAllocation,
maxAMResourcePerQueuePercent, absoluteCapacity);
maxActiveApplicationsPerUser =
CSQueueUtils.computeMaxActiveApplicationsPerUser(
maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);

74
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
View File

@ -158,7 +158,9 @@ public void testLimitsComputation() throws Exception {
int expectedMaxActiveApps =
Math.max(1,
(int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
csConf.getMaximumApplicationMasterResourcePercent() *
csConf.
getMaximumApplicationMasterResourcePerQueuePercent(
queue.getQueuePath()) *
queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications());
@ -183,7 +185,9 @@ public void testLimitsComputation() throws Exception {
expectedMaxActiveApps =
Math.max(1,
(int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
csConf.getMaximumApplicationMasterResourcePercent() *
csConf.
getMaximumApplicationMasterResourcePerQueuePercent(
queue.getQueuePath()) *
queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications());
@ -200,6 +204,72 @@ public void testLimitsComputation() throws Exception {
(int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
);
// should return -1 if per queue setting not set
assertEquals((int)csConf.UNDEFINED, csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
int expectedMaxApps = (int)(csConf.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS *
queue.getAbsoluteCapacity());
assertEquals(expectedMaxApps, queue.getMaxApplications());
int expectedMaxAppsPerUser = (int)(expectedMaxApps *
(queue.getUserLimit()/100.0f) * queue.getUserLimitFactor());
assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
// should default to global setting if per queue setting not set
assertEquals((long) csConf.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
(long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
// Change the per-queue max AM resources percentage.
csConf.setFloat(
"yarn.scheduler.capacity." +
queue.getQueuePath() +
".maximum-am-resource-percent",
0.5f);
// Re-create queues to get new configs.
queues = new HashMap<String, CSQueue>();
root =
CapacityScheduler.parseQueue(csContext, csConf, null, "root",
queues, queues,
CapacityScheduler.queueComparator,
CapacityScheduler.applicationComparator,
TestUtils.spyHook);
clusterResource = Resources.createResource(100 * 16 * GB);
queue = (LeafQueue)queues.get(A);
expectedMaxActiveApps =
Math.max(1,
(int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
csConf.
getMaximumApplicationMasterResourcePerQueuePercent(
queue.getQueuePath()) *
queue.getAbsoluteMaximumCapacity()));
assertEquals((long) 0.5,
(long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications());
// Change the per-queue max applications.
csConf.setInt(
"yarn.scheduler.capacity." +
queue.getQueuePath() +
".maximum-applications", 9999);
// Re-create queues to get new configs.
queues = new HashMap<String, CSQueue>();
root =
CapacityScheduler.parseQueue(csContext, csConf, null, "root",
queues, queues,
CapacityScheduler.queueComparator,
CapacityScheduler.applicationComparator,
TestUtils.spyHook);
queue = (LeafQueue)queues.get(A);
assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
assertEquals(9999, queue.getMaxApplications());
expectedMaxAppsPerUser = (int)(9999 *
(queue.getUserLimit()/100.0f) * queue.getUserLimitFactor());
assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
}
@Test

17
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
View File

@ -240,17 +240,24 @@ Hadoop MapReduce Next Generation - Capacity Scheduler
*--------------------------------------+--------------------------------------+
|| Property || Description |
*--------------------------------------+--------------------------------------+
| <<<yarn.scheduler.capacity.maximum-applications>>> | |
| <<<yarn.scheduler.capacity.maximum-applications>>> / |
| <<<yarn.scheduler.capacity.<queue-path>.maximum-applications>>> | |
| | Maximum number of applications in the system which can be concurrently |
| | active both running and pending. Limits on each queue are directly |
| | proportional to their queue capacities and user limits. This is a
| | hard limit and any applications submitted when this limit is reached will |
| | be rejected. Default is 10000.|
| | be rejected. Default is 10000. This can be set for all queues with |
| | <<<yarn.scheduler.capacity.maximum-applications>>> and can also be overridden on a |
| | per queue basis by setting <<<yarn.scheduler.capacity.<queue-path>.maximum-applications>>>. |
*--------------------------------------+--------------------------------------+
| yarn.scheduler.capacity.maximum-am-resource-percent | |
| <<<yarn.scheduler.capacity.maximum-am-resource-percent>>> / |
| <<<yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent>>> | |
| | Maximum percent of resources in the cluster which can be used to run |
| | application masters - controls number of concurrent running applications. |
| | Specified as a float - ie 0.5 = 50%. Default is 10%. |
| | application masters - controls number of concurrent active applications. Limits on each |
| | queue are directly proportional to their queue capacities and user limits. |
| | Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with |
| | <<<yarn.scheduler.capacity.maximum-am-resource-percent>>> and can also be overridden on a |
| | per queue basis by setting <<<yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent>>> |
*--------------------------------------+--------------------------------------+
* Queue Administration & Permissions