YARN-10872. Replace getPropsWithPrefix calls in AutoCreatedQueueTemplate (#3396)

Co-authored-by: Benjamin Teke <bteke@cloudera.com>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java

@@ -28,11 +28,8 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.util.Comparator;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
 
 /**
  * A container class for automatically created child leaf queues.
@@ -173,44 +170,22 @@ public AutoCreatedLeafQueueConfig getLeafQueueTemplate() {
     return queueManagementPolicy;
   }
 
-  protected SortedMap<String, String> getConfigurationsWithPrefix
-      (SortedMap<String, String> sortedConfigs, String prefix) {
-    return sortedConfigs.subMap( prefix, prefix + Character.MAX_VALUE );
-  }
-
-  protected SortedMap<String, String> sortCSConfigurations() {
-    SortedMap<String, String> sortedConfigs = new TreeMap(
-        new Comparator<String>() {
-          public int compare(String s1, String s2) {
-            return s1.compareToIgnoreCase(s2);
-          }
-
-        });
-
-    for (final Iterator<Map.Entry<String, String>> iterator =
-         csContext.getConfiguration().iterator(); iterator.hasNext(); ) {
-      final Map.Entry<String, String> confKeyValuePair = iterator.next();
-      sortedConfigs.put(confKeyValuePair.getKey(), confKeyValuePair.getValue());
-    }
-    return sortedConfigs;
-  }
-
   protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String
       configPrefix) {
 
     CapacitySchedulerConfiguration leafQueueConfigs = new
         CapacitySchedulerConfiguration(new Configuration(false), false);
 
-    String prefix = YarnConfiguration.RESOURCE_TYPES + ".";
     Map<String, String> rtProps = csContext
-        .getConfiguration().getPropsWithPrefix(prefix);
+        .getConfiguration().getConfigurationProperties()
+        .getPropertiesWithPrefix(YarnConfiguration.RESOURCE_TYPES + ".", true);
     for (Map.Entry<String, String> entry : rtProps.entrySet()) {
-      leafQueueConfigs.set(prefix + entry.getKey(), entry.getValue());
+      leafQueueConfigs.set(entry.getKey(), entry.getValue());
     }
 
-    SortedMap<String, String> sortedConfigs = sortCSConfigurations();
+    Map<String, String> templateConfigs = csContext
-    SortedMap<String, String> templateConfigs = getConfigurationsWithPrefix
+        .getConfiguration().getConfigurationProperties()
-        (sortedConfigs, configPrefix);
+        .getPropertiesWithPrefix(configPrefix, true);
 
     for (final Iterator<Map.Entry<String, String>> iterator =
          templateConfigs.entrySet().iterator(); iterator.hasNext(); ) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java

@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 
 import java.util.ArrayList;
@@ -50,7 +49,7 @@ public class AutoCreatedQueueTemplate {
   private final Map<String, String> leafOnlyProperties = new HashMap<>();
   private final Map<String, String> parentOnlyProperties = new HashMap<>();
 
-  public AutoCreatedQueueTemplate(Configuration configuration,
+  public AutoCreatedQueueTemplate(CapacitySchedulerConfiguration configuration,
                                   String queuePath) {
     setTemplateConfigEntries(configuration, queuePath);
   }
@@ -91,7 +90,7 @@ public Map<String, String> getParentOnlyProperties() {
  * @param conf configuration to set
    * @param childQueuePath child queue path used for prefixing the properties
    */
-  public void setTemplateEntriesForChild(Configuration conf,
+  public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
                                          String childQueuePath) {
     setTemplateEntriesForChild(conf, childQueuePath, false);
   }
@@ -105,16 +104,20 @@ public void setTemplateEntriesForChild(Configuration conf,
    *               parent specific template properties
    * @param childQueuePath child queue path used for prefixing the properties
    */
-  public void setTemplateEntriesForChild(Configuration conf,
+  public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
                                          String childQueuePath,
                                          boolean isLeaf) {
     if (childQueuePath.equals(ROOT)) {
       return;
     }
 
+    ConfigurationProperties configurationProperties =
+        conf.getConfigurationProperties();
+
     // Get all properties that are explicitly set
-    Set<String> alreadySetProps = conf.getPropsWithPrefix(
+    Set<String> alreadySetProps = configurationProperties
-        CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet();
+        .getPropertiesWithPrefix(CapacitySchedulerConfiguration
+            .getQueuePrefix(childQueuePath)).keySet();
 
     // Check template properties only set for leaf or parent queues
     Map<String, String> queueTypeSpecificTemplates = parentOnlyProperties;
@@ -151,8 +154,11 @@ public void setTemplateEntriesForChild(Configuration conf,
    * yarn.scheduler.capacity.root.a.auto-queue-creation-v2.template.capacity
    * yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity
    */
-  private void setTemplateConfigEntries(Configuration configuration,
+  private void setTemplateConfigEntries(CapacitySchedulerConfiguration configuration,
                                         String queuePath) {
+    ConfigurationProperties configurationProperties =
+        configuration.getConfigurationProperties();
+
     List<String> queuePathParts = new ArrayList<>(Arrays.asList(
         queuePath.split("\\.")));
 
@@ -178,8 +184,8 @@ private void setTemplateConfigEntries(Configuration configuration,
       String templateQueuePath = String.join(".", queuePathParts);
       // Get all configuration entries with
       // yarn.scheduler.capacity.<queuePath> prefix
-      Map<String, String> queueProps = configuration.getPropsWithPrefix(
+      Map<String, String> queueProps = configurationProperties
-          getQueuePrefix(templateQueuePath));
+          .getPropertiesWithPrefix(getQueuePrefix(templateQueuePath));
 
       // Store template, parent-template and leaf-template properties
       for (Map.Entry<String, String> entry : queueProps.entrySet()) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java

@@ -1607,7 +1607,8 @@ public Set<String> getConfiguredNodeLabels(String queuePath) {
    */
   public Map<String, Set<String>> getConfiguredNodeLabelsByQueue() {
     Map<String, Set<String>> labelsByQueue = new HashMap<>();
-    Map<String, String> schedulerEntries = getPropsWithPrefix(
+    Map<String, String> schedulerEntries =
+        getConfigurationProperties().getPropertiesWithPrefix(
             CapacitySchedulerConfiguration.PREFIX);
 
     for (Map.Entry<String, String> propertyEntry

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java

@@ -76,7 +76,17 @@ public Map<String, String> getPropertiesWithPrefix(
       String prefix, boolean fullyQualifiedKey) {
     List<String> propertyPrefixParts = splitPropertyByDelimiter(prefix);
     Map<String, String> properties = new HashMap<>();
-    String trimPrefix = fullyQualifiedKey ? "" : prefix;
+    String trimPrefix;
+    if (fullyQualifiedKey) {
+      trimPrefix = "";
+    } else {
+      // To support the behaviour where the
+      // CapacitySchedulerConfiguration.getQueuePrefix(String queue) method
+      // returned with the queue prefix with a dot appended to it the last dot
+      // should be removed
+      trimPrefix = prefix.endsWith(CapacitySchedulerConfiguration.DOT) ?
+          prefix.substring(0, prefix.length() - 1) : prefix;
+    }
 
     collectPropertiesRecursively(nodes, properties,
         propertyPrefixParts.iterator(), trimPrefix);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestConfigurationProperties.java

@@ -58,6 +58,17 @@ public void testGetPropertiesWithPrefix() {
     Assert.assertTrue(props.containsKey("4.5"));
     Assert.assertEquals("TEST_VALUE_3_2", props.get("4.5"));
 
+    // Test the scenario where the prefix has a dot appended to it
+    // (see CapacitySchedulerConfiguration.getQueuePrefix(String queue)).
+    // The dot is disregarded.
+    props = configurationProperties
+        .getPropertiesWithPrefix("root.1.2.4.");
+
+    Assert.assertEquals(2, props.size());
+    Assert.assertTrue(props.containsKey(""));
+    Assert.assertEquals("TEST_VALUE_3_1", props.get(""));
+    Assert.assertTrue(props.containsKey("5"));
+    Assert.assertEquals("TEST_VALUE_3_2", props.get("5"));
 
     Map<String, String> propsWithRootPrefix = configurationProperties
         .getPropertiesWithPrefix("root");