big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11393. Fs2cs could be extended to set ULF to -1 upon conversion (#5201)

Browse Source
This commit is contained in:
susheel-gupta 2023-01-02 20:05:16 +05:30 committed by GitHub
parent b93b1c69cc
commit c44c9f984b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 34 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
View File

@ -17,12 +17,15 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@ -79,6 +82,7 @@ public void convertQueueHierarchy(FSQueue queue) {
emitMaxParallelApps(queueName, queue);
emitMaxAllocations(queueName, queue);
emitPreemptionDisabled(queueName, queue);
emitDefaultUserLimitFactor(queueName, children);
emitChildCapacity(queue);
emitMaximumCapacity(queueName, queue);
@ -215,6 +219,15 @@ private void emitPreemptionDisabled(String queueName, FSQueue queue) {
}
}
public void emitDefaultUserLimitFactor(String queueName, List<FSQueue> children) {
if (children.isEmpty()) {
capacitySchedulerConfig.setFloat(
CapacitySchedulerConfiguration.
PREFIX + queueName + DOT + USER_LIMIT_FACTOR,
-1.0f);
}
}
/**
* yarn.scheduler.fair.sizebasedweight ==>
* yarn.scheduler.capacity.&lt;queue-path&gt;

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
View File

@ -17,6 +17,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.DYNAMIC_MAX_ASSIGN;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CAPACITY_PERCENTAGE;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_CAPACITY;
@ -182,7 +183,26 @@ public void testDefaultMaxAMShare() throws Exception {
conf.get(PREFIX + "root.admins.alice.maximum-am-resource-percent"));
assertNull("root.users.joe maximum-am-resource-percent should be null",
conf.get(PREFIX + "root.users.joe maximum-am-resource-percent"));
conf.get(PREFIX + "root.users.joe.maximum-am-resource-percent"));
}
@Test
public void testDefaultUserLimitFactor() throws Exception {
converter.convert(config);
Configuration conf = converter.getCapacitySchedulerConfig();
assertNull("root.users user-limit-factor should be null",
conf.get(PREFIX + "root.users." + USER_LIMIT_FACTOR));
assertEquals("root.default user-limit-factor", "-1.0",
conf.get(PREFIX + "root.default.user-limit-factor"));
assertEquals("root.users.joe user-limit-factor", "-1.0",
conf.get(PREFIX + "root.users.joe.user-limit-factor"));
assertEquals("root.admins.bob user-limit-factor", "-1.0",
conf.get(PREFIX + "root.admins.bob.user-limit-factor"));
}
@Test