big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11543: Fix checkstyle issues after YARN-11520. (#5896)

Browse Source 
Co-authored-by: Benjamin Teke <bteke@cloudera.com>
This commit is contained in:
Benjamin Teke 2023-07-31 15:44:18 +02:00 committed by GitHub
parent 52d66c2c11
commit 030811bf85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 31 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
View File

@ -439,21 +439,22 @@ protected void setupQueueConfigs(Resource clusterResource) throws
*/ */
protected void parseAndSetDynamicTemplates() { protected void parseAndSetDynamicTemplates() {
// Set the template properties from the parent to the queuepath of the child // Set the template properties from the parent to the queuepath of the child
((AbstractParentQueue) parent).getAutoCreatedQueueTemplate() ((AbstractParentQueue) parent).getAutoCreatedQueueTemplate()
.setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePath(), this instanceof AbstractLeafQueue); .setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePath(),
this instanceof AbstractLeafQueue);
String parentTemplate = String.format("%s.%s", parent.getQueuePath(), String parentTemplate = String.format("%s.%s", parent.getQueuePath(),
AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX); AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX);
parentTemplate = parentTemplate.substring(0, parentTemplate.lastIndexOf( parentTemplate = parentTemplate.substring(0, parentTemplate.lastIndexOf(
DOT)); DOT));
Set<String> parentNodeLabels = queueContext.getQueueManager() Set<String> parentNodeLabels = queueContext.getQueueManager()
.getConfiguredNodeLabelsForAllQueues() .getConfiguredNodeLabelsForAllQueues()
.getLabelsByQueue(parentTemplate); .getLabelsByQueue(parentTemplate);
if (parentNodeLabels != null && parentNodeLabels.size() > 1) { if (parentNodeLabels != null && parentNodeLabels.size() > 1) {
queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues() queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues()
.setLabelsByQueue(getQueuePath(), new HashSet<>(parentNodeLabels)); .setLabelsByQueue(getQueuePath(), new HashSet<>(parentNodeLabels));
} }
} }
protected void setDynamicQueueACLProperties() { protected void setDynamicQueueACLProperties() {

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors.java
View File

@ -35,7 +35,7 @@
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
public class TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors public class TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors
extends TestCapacitySchedulerAutoCreatedQueueBase { extends TestCapacitySchedulerAutoCreatedQueueBase {
public static final int GB = 1024; public static final int GB = 1024;
private static final String NL = CommonNodeLabelsManager.NO_LABEL; private static final String NL = CommonNodeLabelsManager.NO_LABEL;
private MockRM mockRM = null; private MockRM mockRM = null;
@ -112,8 +112,10 @@ private void createAbsoluteConfig() {
csConf.setAutoQueueCreationV2Enabled(A, true); csConf.setAutoQueueCreationV2Enabled(A, true);
// Set up dynamic queue templates // Set up dynamic queue templates
csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"), "[memory=2000mb, vcores=2]"); csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"),
csConf.set(getLeafTemplateKey(A, "capacity"), "[memory=2000, vcores=4]"); "[memory=2000mb, vcores=2]");
csConf.set(getLeafTemplateKey(A, "capacity"),
"[memory=2000, vcores=4]");
} }
private void createMixedConfig() { private void createMixedConfig() {
@ -140,9 +142,12 @@ private void createMixedConfig() {
csConf.setAutoQueueCreationV2Enabled(D, true); csConf.setAutoQueueCreationV2Enabled(D, true);
// Set up dynamic queue templates // Set up dynamic queue templates
csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"), "[memory=2w, vcores=5w]"); csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"),
csConf.set(getParentTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"), "[memory=2w, vcores=10]"); "[memory=2w, vcores=5w]");
csConf.set(getLeafTemplateKey(CapacitySchedulerConfiguration.ROOT + ".*", "capacity"), "[memory=2000, vcores=2]"); csConf.set(getParentTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"),
"[memory=2w, vcores=10]");
csConf.set(getLeafTemplateKey(CapacitySchedulerConfiguration.ROOT + ".*", "capacity"),
"[memory=2000, vcores=2]");
csConf.set(getLeafTemplateKey(D, "capacity"), "[memory=1000, vcores=1]"); csConf.set(getLeafTemplateKey(D, "capacity"), "[memory=1000, vcores=1]");
} }
@ -214,7 +219,8 @@ private void validateBasicConfiguration() throws Exception {
} }
/* /*
Create and validate the following structure with mixed resource vectors and non-legacy queue mode: Create and validate the following structure with mixed resource vectors
and non-legacy queue mode:
root root
┌─────┬────────┬─────┴─────┬─────────┐ ┌─────┬────────┬─────┴─────┬─────────┐
@ -232,8 +238,9 @@ public void testMixedFlexibleConfiguration() throws Exception {
// Check if queue c-auto got created // Check if queue c-auto got created
CSQueue cAuto = cs.getQueue("root.c-auto"); CSQueue cAuto = cs.getQueue("root.c-auto");
// At this point queues a, b, d exists, ant c-auto was just created // At this point queues a, b, d exists, ant c-auto was just created
// b takes 2000 MB from the cluster, a and d take up 10 + 10 = 20% (6000 MB, 6 vcore), so c-auto should get the rest // b takes 2000 MB from the cluster, a and d take up 10 + 10 = 20% (6000 MB, 6 vcore),
// (24000 MB, 24 vcore) because it's the only one with configured weights // so c-auto should get the rest (24000 MB, 24 vcore) because it's the only one
// with configured weights
Assert.assertEquals(24 / 32f, cAuto.getAbsoluteCapacity(), EPSILON); Assert.assertEquals(24 / 32f, cAuto.getAbsoluteCapacity(), EPSILON);
Assert.assertEquals(-1f, cAuto.getQueueCapacities().getWeight(), EPSILON); Assert.assertEquals(-1f, cAuto.getQueueCapacities().getWeight(), EPSILON);
Assert.assertEquals(24000, Assert.assertEquals(24000,
@ -267,7 +274,8 @@ public void testMixedFlexibleConfiguration() throws Exception {
Assert.assertEquals(2, Assert.assertEquals(2,
a2Auto.getQueueResourceQuotas().getEffectiveMinResource().getVirtualCores()); a2Auto.getQueueResourceQuotas().getEffectiveMinResource().getVirtualCores());
// Absolute requests take precedence over percentage and weight, hence a1 should have 1000 MB, 0 vcore // Absolute requests take precedence over percentage and weight,
// hence a1 should have 1000 MB, 0 vcore
CSQueue a1 = cs.getQueue("root.a.a1"); CSQueue a1 = cs.getQueue("root.a.a1");
Assert.assertEquals(1000, Assert.assertEquals(1000,
a1.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); a1.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize());