big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10599. fs2cs should generate new 'auto-queue-creation-v2.enabled' properties for all parents. Contributed by Peter Bacsko

Browse Source
This commit is contained in:
Szilard Nemeth 2021-01-27 15:49:26 +01:00
parent 80c7404b51
commit 7c4ef42837
4 changed files with 75 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
View File

@ -23,6 +23,7 @@
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
public class WeightToWeightConverter
@ -33,14 +34,21 @@ public void convertWeightsForChildQueues(FSQueue queue,
Configuration csConfig) {
List<FSQueue> children = queue.getChildQueues();
if (queue instanceof FSParentQueue || !children.isEmpty()) {
children.forEach(fsQueue -> csConfig.set(
getProperty(fsQueue), getWeightString(fsQueue)));
csConfig.setBoolean(getAutoCreateV2EnabledProperty(queue), true);
}
}
private String getProperty(FSQueue queue) {
return PREFIX + queue.getName() + ".capacity";
}
private String getAutoCreateV2EnabledProperty(FSQueue queue) {
return PREFIX + queue.getName() + ".auto-queue-creation-v2.enabled";
}
private String getWeightString(FSQueue queue) {
return Float.toString(queue.getWeight()) + "w";
}

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
View File

@ -334,6 +334,35 @@ public void testChildCapacityInWeightMode() {
csConfig.get(PREFIX + "root.misc.b.capacity"));
}
@Test
public void testAutoCreateV2FlagsInWeightMode() {
converter = builder.withPercentages(false).build();
converter.convertQueueHierarchy(rootQueue);
assertTrue("root autocreate v2 flag",
csConfig.getBoolean(
PREFIX + "root.auto-queue-creation-v2.enabled", false));
assertTrue("root.admins autocreate v2 flag",
csConfig.getBoolean(
PREFIX + "root.admins.auto-queue-creation-v2.enabled", false));
assertTrue("root.users autocreate v2 flag",
csConfig.getBoolean(
PREFIX + "root.users.auto-queue-creation-v2.enabled", false));
assertTrue("root.misc autocreate v2 flag",
csConfig.getBoolean(
PREFIX + "root.misc.auto-queue-creation-v2.enabled", false));
Set<String> leafs = Sets.difference(ALL_QUEUES,
Sets.newHashSet("root",
"root.default",
"root.admins",
"root.users",
"root.misc"));
assertNoValueForQueues(leafs, "auto-queue-creation-v2.enabled",
csConfig);
}
@Test
public void testZeroSumCapacityValidation() {
converter = builder.withPercentages(true).build();

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java
View File

@ -20,6 +20,9 @@
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@ -41,7 +44,7 @@ public void testNoChildQueueConversion() {
FSQueue root = createFSQueues();
converter.convertWeightsForChildQueues(root, config);
assertEquals("Converted items", 0,
assertEquals("Converted items", 1,
config.getPropsWithPrefix(PREFIX).size());
}
@ -52,6 +55,8 @@ public void testSingleWeightConversion() {
assertEquals("root.a weight", "1.0w",
config.get(PREFIX + "root.a.capacity"));
assertEquals("Number of properties", 2,
config.getPropsWithPrefix(PREFIX).size());
}
@Test
@ -60,7 +65,7 @@ public void testMultiWeightConversion() {
converter.convertWeightsForChildQueues(root, config);
assertEquals("Number of properties", 3,
assertEquals("Number of properties", 4,
config.getPropsWithPrefix(PREFIX).size());
assertEquals("root.a weight", "1.0w",
config.get(PREFIX + "root.a.capacity"));
@ -69,4 +74,26 @@ public void testMultiWeightConversion() {
assertEquals("root.c weight", "3.0w",
config.get(PREFIX + "root.c.capacity"));
}
@Test
public void testAutoCreateV2FlagOnParent() {
FSQueue root = createFSQueues(1);
converter.convertWeightsForChildQueues(root, config);
assertTrue("root autocreate v2 enabled",
config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled",
false));
}
@Test
public void testAutoCreateV2FlagOnParentWithoutChildren() {
FSQueue root = createParent(new ArrayList<>());
converter.convertWeightsForChildQueues(root, config);
assertEquals("Number of properties", 1,
config.getPropsWithPrefix(PREFIX).size());
assertTrue("root autocreate v2 enabled",
config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled",
false));
}
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java
View File

@ -24,6 +24,7 @@
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.util.resource.Resources;
@ -44,12 +45,15 @@ protected FSQueue createFSQueues(int... weights){
queues.add(queue);
}
FSQueue root = mock(FSQueue.class);
return createParent(queues);
}
protected FSParentQueue createParent(List<FSQueue> children) {
FSParentQueue root = mock(FSParentQueue.class);
when(root.getWeight()).thenReturn(1.0f);
when(root.getName()).thenReturn("root");
when(root.getMinShare()).thenReturn(Resources.none());
when(root.getChildQueues()).thenReturn(queues);
when(root.getChildQueues()).thenReturn(children);
return root;
}
}