big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9569. Auto-created leaf queues do not honor cluster-wide min/max memory/vcores. Contributed by Craig Condit.

Browse Source
This commit is contained in:
Suma Shivaprasad 2019-06-10 14:33:24 -07:00
parent 0d160a0ba8
commit 9191e08f0a
3 changed files with 38 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
View File

@ -19,6 +19,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common
@ -200,6 +201,13 @@ protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String
CapacitySchedulerConfiguration leafQueueConfigs = new
CapacitySchedulerConfiguration(new Configuration(false), false);
String prefix = YarnConfiguration.RESOURCE_TYPES + ".";
Map<String, String> rtProps = csContext
.getConfiguration().getPropsWithPrefix(prefix);
for (Map.Entry<String, String> entry : rtProps.entrySet()) {
leafQueueConfigs.set(prefix + entry.getKey(), entry.getValue());
}
SortedMap<String, String> sortedConfigs = sortCSConfigurations();
SortedMap<String, String> templateConfigs = getConfigurationsWithPrefix
(sortedConfigs, configPrefix);

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
View File

@ -22,6 +22,7 @@
import org.apache.commons.lang3.RandomUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.GroupMappingServiceProvider;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
@ -189,7 +190,7 @@ void spyOnNextEvent(Event expectedEvent, long timeout)
@Before
public void setUp() throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
CapacitySchedulerConfiguration conf = setupSchedulerConfiguration();
setupQueueConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
@ -494,8 +495,22 @@ protected List<UserGroupMappingPlacementRule.QueueMapping> setupQueueMapping(
return queueMappings;
}
protected CapacitySchedulerConfiguration setupSchedulerConfiguration() {
Configuration schedConf = new Configuration();
schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ ".vcores.minimum-allocation", 1);
schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ ".vcores.maximum-allocation", 8);
schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ ".memory-mb.minimum-allocation", 1024);
schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ ".memory-mb.maximum-allocation", 16384);
return new CapacitySchedulerConfiguration(schedConf);
}
protected MockRM setupSchedulerInstance() throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
CapacitySchedulerConfiguration conf = setupSchedulerConfiguration();
setupQueueConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
@ -579,6 +594,14 @@ protected void validateUserAndAppLimits(
autoCreatedLeafQueue.getMaxApplicationsPerUser());
}
protected void validateContainerLimits(
AutoCreatedLeafQueue autoCreatedLeafQueue) {
assertEquals(8,
autoCreatedLeafQueue.getMaximumAllocation().getVirtualCores());
assertEquals(16384,
autoCreatedLeafQueue.getMaximumAllocation().getMemorySize());
}
protected void validateInitialQueueEntitlement(CSQueue parentQueue, String
leafQueueName, Map<String, Float>
expectedTotalChildQueueAbsCapacityByLabel,

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
View File

@ -132,6 +132,7 @@ public void testAutoCreateLeafQueueCreation() throws Exception {
expectedChildQueueAbsCapacity, accessibleNodeLabelsOnC);
validateUserAndAppLimits(autoCreatedLeafQueue, 1000, 1000);
validateContainerLimits(autoCreatedLeafQueue);
assertTrue(autoCreatedLeafQueue
.getOrderingPolicy() instanceof FairOrderingPolicy);
@ -256,8 +257,7 @@ public void testConvertAutoCreateDisabledOnManagedParentQueueFails()
throws Exception {
CapacityScheduler newCS = new CapacityScheduler();
try {
CapacitySchedulerConfiguration newConf =
new CapacitySchedulerConfiguration();
CapacitySchedulerConfiguration newConf = setupSchedulerConfiguration();
setupQueueConfiguration(newConf);
newConf.setAutoCreateChildQueueEnabled(C, false);
@ -285,8 +285,7 @@ public void testConvertLeafQueueToParentQueueWithAutoCreate()
throws Exception {
CapacityScheduler newCS = new CapacityScheduler();
try {
CapacitySchedulerConfiguration newConf =
new CapacitySchedulerConfiguration();
CapacitySchedulerConfiguration newConf = setupSchedulerConfiguration();
setupQueueConfiguration(newConf);
newConf.setAutoCreatedLeafQueueConfigCapacity(A1, A1_CAPACITY / 10);
newConf.setAutoCreateChildQueueEnabled(A1, true);
@ -315,8 +314,7 @@ public void testConvertFailsFromParentQueueToManagedParentQueue()
throws Exception {
CapacityScheduler newCS = new CapacityScheduler();
try {
CapacitySchedulerConfiguration newConf =
new CapacitySchedulerConfiguration();
CapacitySchedulerConfiguration newConf = setupSchedulerConfiguration();
setupQueueConfiguration(newConf);
newConf.setAutoCreatedLeafQueueConfigCapacity(A, A_CAPACITY / 10);
newConf.setAutoCreateChildQueueEnabled(A, true);
@ -773,6 +771,7 @@ public void testReinitializeQueuesWithAutoCreatedLeafQueues()
validateCapacities(user3Queue, 0.3f, 0.09f, 0.4f,0.2f);
validateUserAndAppLimits(user3Queue, 900, 900);
validateContainerLimits(user3Queue);
GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy =
(GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue)