big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10872. Replace getPropsWithPrefix calls in AutoCreatedQueueTemplate (#3396)

Browse Source 
Co-authored-by: Benjamin Teke <bteke@cloudera.com>
This commit is contained in:
Benjamin Teke 2021-09-10 17:32:42 +02:00 committed by GitHub
parent 811fd23f23
commit 971f1b8b0a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 46 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
View File

@ -28,11 +28,8 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
/**
* A container class for automatically created child leaf queues.
@ -173,44 +170,22 @@ public AutoCreatedLeafQueueConfig getLeafQueueTemplate() {
return queueManagementPolicy;
}
protected SortedMap<String, String> getConfigurationsWithPrefix
(SortedMap<String, String> sortedConfigs, String prefix) {
return sortedConfigs.subMap( prefix, prefix + Character.MAX_VALUE );
}
protected SortedMap<String, String> sortCSConfigurations() {
SortedMap<String, String> sortedConfigs = new TreeMap(
new Comparator<String>() {
public int compare(String s1, String s2) {
return s1.compareToIgnoreCase(s2);
}
});
for (final Iterator<Map.Entry<String, String>> iterator =
csContext.getConfiguration().iterator(); iterator.hasNext(); ) {
final Map.Entry<String, String> confKeyValuePair = iterator.next();
sortedConfigs.put(confKeyValuePair.getKey(), confKeyValuePair.getValue());
}
return sortedConfigs;
}
protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String
configPrefix) {
CapacitySchedulerConfiguration leafQueueConfigs = new
CapacitySchedulerConfiguration(new Configuration(false), false);
String prefix = YarnConfiguration.RESOURCE_TYPES + ".";
Map<String, String> rtProps = csContext
.getConfiguration().getPropsWithPrefix(prefix);
.getConfiguration().getConfigurationProperties()
.getPropertiesWithPrefix(YarnConfiguration.RESOURCE_TYPES + ".", true);
for (Map.Entry<String, String> entry : rtProps.entrySet()) {
leafQueueConfigs.set(prefix + entry.getKey(), entry.getValue());
leafQueueConfigs.set(entry.getKey(), entry.getValue());
}
SortedMap<String, String> sortedConfigs = sortCSConfigurations();
SortedMap<String, String> templateConfigs = getConfigurationsWithPrefix
(sortedConfigs, configPrefix);
Map<String, String> templateConfigs = csContext
.getConfiguration().getConfigurationProperties()
.getPropertiesWithPrefix(configPrefix, true);
for (final Iterator<Map.Entry<String, String>> iterator =
templateConfigs.entrySet().iterator(); iterator.hasNext(); ) {

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
@ -50,7 +49,7 @@ public class AutoCreatedQueueTemplate {
private final Map<String, String> leafOnlyProperties = new HashMap<>();
private final Map<String, String> parentOnlyProperties = new HashMap<>();
public AutoCreatedQueueTemplate(Configuration configuration,
public AutoCreatedQueueTemplate(CapacitySchedulerConfiguration configuration,
String queuePath) {
setTemplateConfigEntries(configuration, queuePath);
}
@ -91,7 +90,7 @@ public Map<String, String> getParentOnlyProperties() {
* @param conf configuration to set
* @param childQueuePath child queue path used for prefixing the properties
*/
public void setTemplateEntriesForChild(Configuration conf,
public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
String childQueuePath) {
setTemplateEntriesForChild(conf, childQueuePath, false);
}
@ -105,16 +104,20 @@ public void setTemplateEntriesForChild(Configuration conf,
* parent specific template properties
* @param childQueuePath child queue path used for prefixing the properties
*/
public void setTemplateEntriesForChild(Configuration conf,
public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
String childQueuePath,
boolean isLeaf) {
if (childQueuePath.equals(ROOT)) {
return;
}
ConfigurationProperties configurationProperties =
conf.getConfigurationProperties();
// Get all properties that are explicitly set
Set<String> alreadySetProps = conf.getPropsWithPrefix(
CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet();
Set<String> alreadySetProps = configurationProperties
.getPropertiesWithPrefix(CapacitySchedulerConfiguration
.getQueuePrefix(childQueuePath)).keySet();
// Check template properties only set for leaf or parent queues
Map<String, String> queueTypeSpecificTemplates = parentOnlyProperties;
@ -151,8 +154,11 @@ public void setTemplateEntriesForChild(Configuration conf,
* yarn.scheduler.capacity.root.a.auto-queue-creation-v2.template.capacity
* yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity
*/
private void setTemplateConfigEntries(Configuration configuration,
private void setTemplateConfigEntries(CapacitySchedulerConfiguration configuration,
String queuePath) {
ConfigurationProperties configurationProperties =
configuration.getConfigurationProperties();
List<String> queuePathParts = new ArrayList<>(Arrays.asList(
queuePath.split("\\.")));
@ -178,8 +184,8 @@ private void setTemplateConfigEntries(Configuration configuration,
String templateQueuePath = String.join(".", queuePathParts);
// Get all configuration entries with
// yarn.scheduler.capacity.<queuePath> prefix
Map<String, String> queueProps = configuration.getPropsWithPrefix(
getQueuePrefix(templateQueuePath));
Map<String, String> queueProps = configurationProperties
.getPropertiesWithPrefix(getQueuePrefix(templateQueuePath));
// Store template, parent-template and leaf-template properties
for (Map.Entry<String, String> entry : queueProps.entrySet()) {

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -1607,7 +1607,8 @@ public Set<String> getConfiguredNodeLabels(String queuePath) {
*/
public Map<String, Set<String>> getConfiguredNodeLabelsByQueue() {
Map<String, Set<String>> labelsByQueue = new HashMap<>();
Map<String, String> schedulerEntries = getPropsWithPrefix(
Map<String, String> schedulerEntries =
getConfigurationProperties().getPropertiesWithPrefix(
CapacitySchedulerConfiguration.PREFIX);
for (Map.Entry<String, String> propertyEntry

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java
View File

@ -76,7 +76,17 @@ public Map<String, String> getPropertiesWithPrefix(
String prefix, boolean fullyQualifiedKey) {
List<String> propertyPrefixParts = splitPropertyByDelimiter(prefix);
Map<String, String> properties = new HashMap<>();
String trimPrefix = fullyQualifiedKey ? "" : prefix;
String trimPrefix;
if (fullyQualifiedKey) {
trimPrefix = "";
} else {
// To support the behaviour where the
// CapacitySchedulerConfiguration.getQueuePrefix(String queue) method
// returned with the queue prefix with a dot appended to it the last dot
// should be removed
trimPrefix = prefix.endsWith(CapacitySchedulerConfiguration.DOT) ?
prefix.substring(0, prefix.length() - 1) : prefix;
}
collectPropertiesRecursively(nodes, properties,
propertyPrefixParts.iterator(), trimPrefix);

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestConfigurationProperties.java
View File

@ -58,6 +58,17 @@ public void testGetPropertiesWithPrefix() {
Assert.assertTrue(props.containsKey("4.5"));
Assert.assertEquals("TEST_VALUE_3_2", props.get("4.5"));
// Test the scenario where the prefix has a dot appended to it
// (see CapacitySchedulerConfiguration.getQueuePrefix(String queue)).
// The dot is disregarded.
props = configurationProperties
.getPropertiesWithPrefix("root.1.2.4.");
Assert.assertEquals(2, props.size());
Assert.assertTrue(props.containsKey(""));
Assert.assertEquals("TEST_VALUE_3_1", props.get(""));
Assert.assertTrue(props.containsKey("5"));
Assert.assertEquals("TEST_VALUE_3_2", props.get("5"));
Map<String, String> propsWithRootPrefix = configurationProperties
.getPropertiesWithPrefix("root");