YARN-11641. Fix getCapacityConfigurationTypeForQueues for absolute mode with zero resource. (#6435)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java

@@ -229,8 +229,7 @@ public QueueCapacityType getCapacityConfigurationTypeForQueues(
               "{Queue=" + queue.getQueuePath() + ", label=" + nodeLabel
                   + " uses weight mode}. ");
         }
-        if (!queue.getQueueResourceQuotas().getConfiguredMinResource(nodeLabel)
-            .equals(Resources.none())) {
+        if (checkConfigTypeIsAbsoluteResource(queue.getQueuePath(), nodeLabel)) {
           absoluteMinResSet = true;
           // There's a special handling: when absolute resource is configured,
           // capacity will be calculated (and set) for UI/metrics purposes, so

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java

@@ -95,7 +95,7 @@ public void testPercentageMode() throws Exception {
     conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "12.5");
     conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "12.5");
     conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "75");
-    try (MockRM rm = createMutableRM(createConfiguration(conf))) {
+    try (MockRM rm = createMutableRM(createConfiguration(conf), false)) {
       runTest(EXPECTED_FILE_TMPL, "testPercentageMode", rm, resource());
     }
   }
@@ -111,7 +111,7 @@ public void testAbsoluteMode() throws Exception {
     conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "[memory=2048,vcores=2]");
     conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "[memory=2048,vcores=2]");
     conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "[memory=12288,vcores=12]");
-    try (MockRM rm = createMutableRM(createConfiguration(conf))) {
+    try (MockRM rm = createMutableRM(createConfiguration(conf), false)) {
       runTest(EXPECTED_FILE_TMPL, "testAbsoluteMode", rm, resource());
     }
   }
@@ -128,7 +128,7 @@ public void testWeightMode() throws Exception {
     conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "2w");
     conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "2w");
     conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "12w");
-    try (MockRM rm = createMutableRM(createConfiguration(conf))) {
+    try (MockRM rm = createMutableRM(createConfiguration(conf), false)) {
       // capacity and normalizedWeight are set differently between legacy/non-legacy queue mode
       runTest(EXPECTED_FILE_TMPL, "testWeightMode", rm, resource());
     }
@@ -149,7 +149,7 @@ public void testWeightModeFlexibleAQC() throws Exception {
 
     Configuration config = createConfiguration(conf);
     setupAQC(config, "yarn.scheduler.capacity.root.test2.");
-    try (MockRM rm = createMutableRM(config)) {
+    try (MockRM rm = createMutableRM(config, false)) {
       // capacity and normalizedWeight are set differently between legacy/non-legacy queue mode
       rm.registerNode("h1:1234", 32 * GB, 32);
       assertJsonResponse(sendRequest(resource()),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java

@@ -77,7 +77,7 @@ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
     conf.put("yarn.scheduler.capacity.root.managed.capacity", "75");
     conf.put("yarn.scheduler.capacity.root.managed." +
         "auto-create-child-queue.enabled", "true");
-    try (MockRM rm = createMutableRM(createConfiguration(conf))) {
+    try (MockRM rm = createMutableRM(createConfiguration(conf), false)) {
       rm.registerNode("h1:1234", 32 * GB, 32);
       assertJsonResponse(sendRequest(),
           "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
@@ -100,7 +100,7 @@ public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
         "user");
     conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.acl_administer_queue",
         "admin");
-    try (MockRM rm = createMutableRM(createConfiguration(conf))) {
+    try (MockRM rm = createMutableRM(createConfiguration(conf), false)) {
       rm.registerNode("h1:1234", 32 * GB, 32);
       CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
       CapacitySchedulerQueueManager autoQueueHandler = cs.getCapacitySchedulerQueueManager();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedulerConfigMutation.java

@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.sun.jersey.api.client.ClientResponse;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.backupSchedulerConfigFileInTarget;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.getCapacitySchedulerConfigFileInTarget;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.getExpectedResourceFile;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.restoreSchedulerConfigFileInTarget;
+import static org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils.toJson;
+import static org.junit.Assert.assertEquals;
+
+@RunWith(Parameterized.class)
+public class TestRMWebServicesCapacitySchedulerConfigMutation extends JerseyTestBase {
+  private static final String EXPECTED_FILE_TMPL = "webapp/configmutation-%s-%s.json";
+  private final boolean legacyQueueMode;
+  private final String userName;
+
+  @Parameterized.Parameters(name = "{index}: legacy-queue-mode={0}")
+  public static Collection<Boolean> getParameters() {
+    return Arrays.asList(true, false);
+  }
+
+  @BeforeClass
+  public static void beforeClass() {
+    backupSchedulerConfigFileInTarget();
+  }
+
+  @AfterClass
+  public static void afterClass() {
+    restoreSchedulerConfigFileInTarget();
+  }
+
+  public TestRMWebServicesCapacitySchedulerConfigMutation(boolean legacyQueueMode)
+      throws IOException {
+    super(createWebAppDescriptor());
+    this.legacyQueueMode = legacyQueueMode;
+    userName = UserGroupInformation.getCurrentUser().getShortUserName();
+  }
+
+  @Test
+  public void testUpdateAbsoluteHierarchyWithZeroCapacities() throws Exception {
+    Configuration absoluteConfig = createAbsoluteConfig();
+    FileOutputStream out = new FileOutputStream(getCapacitySchedulerConfigFileInTarget());
+    absoluteConfig.writeXml(out);
+    out.close();
+
+    try (MockRM rm = createMutableRM(absoluteConfig, true)){
+      rm.registerNode("h1:1234", 32 * GB, 32);
+
+      assertJsonResponse(resource().path("ws/v1/cluster/scheduler")
+              .queryParam("user.name", userName)
+              .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class),
+          getExpectedResourceFile(EXPECTED_FILE_TMPL, "absolute-hierarchy", "before-update",
+              legacyQueueMode));
+
+      SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+      Map<String, String> capacityChange = new HashMap<>();
+      capacityChange.put(CapacitySchedulerConfiguration.CAPACITY,
+          "[memory=4096, vcores=4]");
+      capacityChange.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY,
+          "[memory=32768, vcores=32]");
+      QueueConfigInfo b = new QueueConfigInfo("root.a", capacityChange);
+      updateInfo.getUpdateQueueInfo().add(b);
+
+      ClientResponse response = resource().path("ws/v1/cluster/scheduler-conf")
+          .queryParam("user.name", userName)
+          .accept(MediaType.APPLICATION_JSON)
+          .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+          .put(ClientResponse.class);
+
+      // HTTP 400 - Bad Request is encountered, check the logs for the failure
+      assertEquals(Response.Status.OK.getStatusCode(), response.getStatus());
+
+      assertJsonResponse(resource().path("ws/v1/cluster/scheduler")
+              .queryParam("user.name", userName)
+              .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class),
+          getExpectedResourceFile(EXPECTED_FILE_TMPL, "absolute-hierarchy", "after-update",
+              legacyQueueMode));
+    }
+  }
+
+  private Configuration createAbsoluteConfig() {
+    Configuration conf = new Configuration(false);
+    conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
+    conf.set("yarn.scheduler.capacity.legacy-queue-mode.enabled", String.valueOf(legacyQueueMode));
+    conf.set("yarn.scheduler.capacity.root.capacity", "[memory=32768, vcores=32]");
+    conf.set("yarn.scheduler.capacity.root.queues", "default, a");
+    conf.set("yarn.scheduler.capacity.root.default.capacity", "[memory=1024, vcores=1]");
+    conf.set("yarn.scheduler.capacity.root.a.capacity", "[memory=0, vcores=0]");
+    conf.set("yarn.scheduler.capacity.root.a.max-capacity", "[memory=32768, vcores=32]");
+    conf.set("yarn.scheduler.capacity.root.a.queues", "b, c");
+    conf.set("yarn.scheduler.capacity.root.a.b.capacity", "[memory=0, vcores=0]");
+    conf.set("yarn.scheduler.capacity.root.a.b.max-capacity", "[memory=32768, vcores=32]");
+    conf.set("yarn.scheduler.capacity.root.a.c.capacity", "[memory=0, vcores=0]");
+    conf.set("yarn.scheduler.capacity.root.a.c.max-capacity", "[memory=32768, vcores=32]");
+    return conf;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java

@@ -115,7 +115,7 @@ public void setUp() throws Exception {
         createConfig(new CapacitySchedulerConfiguration(new Configuration(false)));
     rm = createMockRM(config);
     GuiceServletConfig.setInjector(
-        Guice.createInjector(new TestWebServiceUtil.WebServletModule(rm)));
+        Guice.createInjector(new TestWebServiceUtil.WebServletModule(rm, false)));
   }
 
   public static MockRM createMockRM(CapacitySchedulerConfiguration csConf) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestWebServiceUtil.java

@@ -87,9 +87,11 @@ private TestWebServiceUtil(){
 
   public static class WebServletModule extends ServletModule {
     private final MockRM rm;
+    private final boolean setCustomAuthFilter;
 
-    WebServletModule(MockRM rm) {
+    WebServletModule(MockRM rm, boolean setCustomAuthFilter) {
       this.rm = rm;
+      this.setCustomAuthFilter = setCustomAuthFilter;
     }
 
     @Override
@@ -99,6 +101,11 @@ protected void configureServlets() {
       bind(GenericExceptionHandler.class);
       bind(ResourceManager.class).toInstance(rm);
       serve("/*").with(GuiceContainer.class);
+
+      if (setCustomAuthFilter) {
+        filter("/*").through(TestRMWebServicesAppsModification
+            .TestRMCustomAuthFilter.class);
+      }
     }
   }
 
@@ -321,20 +328,25 @@ public static WebAppDescriptor createWebAppDescriptor() {
   }
 
   public static MockRM createRM(Configuration config) {
+    return createRM(config, false);
+  }
+
+  public static MockRM createRM(Configuration config, boolean setCustomAuthFilter) {
     config.setClass(YarnConfiguration.RM_SCHEDULER,
         CapacityScheduler.class, ResourceScheduler.class);
     config.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
         YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
     MockRM rm = new MockRM(config);
-    GuiceServletConfig.setInjector(Guice.createInjector(new WebServletModule(rm)));
+    GuiceServletConfig.setInjector(Guice.createInjector(
+        new WebServletModule(rm, setCustomAuthFilter)));
     rm.start();
     return rm;
   }
 
-  public static MockRM createMutableRM(Configuration conf) {
+  public static MockRM createMutableRM(Configuration conf, boolean setCustomAuthFilter) {
     conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
         YarnConfiguration.MEMORY_CONFIGURATION_STORE);
-    return createRM(new CapacitySchedulerConfiguration(conf));
+    return createRM(new CapacitySchedulerConfiguration(conf), setCustomAuthFilter);
   }
 
   public static void reinitialize(MockRM rm, Configuration conf) throws IOException {