diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index fd144f23cd..c02e1d3ea1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -128,7 +128,7 @@ public abstract class AbstractCSQueue implements CSQueue { // either at this level or anywhere in the queue's hierarchy. private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false; - protected enum CapacityConfigType { + public enum CapacityConfigType { // FIXME, from what I can see, Percentage mode can almost apply to weighted // and percentage mode at the same time, there's only small area need to be // changed, we need to rename "PERCENTAGE" to "PERCENTAGE" and "WEIGHT" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 8de752901f..9794acd34f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper; import java.util.ArrayList; import java.util.Map; @@ -53,6 +54,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected QueueAclsInfo queueAcls; protected int queuePriority; protected String orderingPolicyInfo; + protected String mode; @XmlTransient static final float EPSILON = 1e-8f; @@ -98,6 +100,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { orderingPolicyInfo = ((ParentQueue) parent).getQueueOrderingPolicy() .getConfigName(); } + mode = CapacitySchedulerInfoHelper.getMode(parent); } public float getCapacity() { @@ -173,4 +176,9 @@ protected CapacitySchedulerQueueInfoList getQueues( } return queuesInfo; } + + public String getMode() { + return mode; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 897f94043a..6fb0290b1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity. CapacitySchedulerConfiguration.RESOURCE_PATTERN; @@ -86,6 +87,7 @@ public class CapacitySchedulerQueueInfo { protected String orderingPolicyInfo; protected boolean autoCreateChildQueueEnabled; protected LeafQueueTemplateInfo leafQueueTemplate; + protected String mode; CapacitySchedulerQueueInfo() { }; @@ -128,6 +130,8 @@ public class CapacitySchedulerQueueInfo { QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas(); populateQueueCapacities(qCapacities, qResQuotas); + mode = CapacitySchedulerInfoHelper.getMode(q); + ResourceUsage queueResourceUsage = q.getQueueResourceUsage(); populateQueueResourceUsage(queueResourceUsage); @@ -306,4 +310,8 @@ public boolean isAutoCreateChildQueueEnabled() { public LeafQueueTemplateInfo getLeafQueueTemplate() { return leafQueueTemplate; } + + public String getMode() { + return mode; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java new file mode 100644 index 0000000000..9727f9ffd6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; + +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; + +public class CapacitySchedulerInfoHelper { + + private CapacitySchedulerInfoHelper() {} + + public static String getMode(CSQueue queue) throws YarnRuntimeException { + if (queue.getCapacityConfigType() == + AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE) { + return "absolute"; + } else if (queue.getCapacityConfigType() == + AbstractCSQueue.CapacityConfigType.PERCENTAGE) { + float weight = queue.getQueueCapacities().getWeight(); + if (weight == -1) { + //-1 indicates we are not in weight mode + return "percentage"; + } else { + return "weight"; + } + } + throw new YarnRuntimeException("Unknown mode for queue: " + + queue.getQueuePath() + ". Queue details: " + queue); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/package-info.java new file mode 100644 index 0000000000..dcfaef9406 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/package-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains helper classes for any Info object. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java index 1477a33676..873e3b95a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java @@ -46,7 +46,7 @@ public static Configuration createBasicCSConfiguration() { conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING"); conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING"); conf.put("yarn.scheduler.capacity.queue-mappings", - "u:test1:test1,u:test2:test2"); + "u:test1:root.test1,u:test2:root.test2"); return createConfiguration(conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 1e856bd412..9fbd2178c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -362,7 +362,7 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException, JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements in: " + info, 12, info.length()); + assertEquals("incorrect number of elements in: " + info, 13, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), @@ -413,10 +413,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 27; + int numExpectedElements = 28; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 45; + numExpectedElements = 46; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java new file mode 100644 index 0000000000..8ee00296e6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -0,0 +1,274 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.ws.rs.core.MediaType; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.JettyUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.GuiceServletConfig; +import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; + +public class TestRMWebServicesCapacitySchedDynamicConfig extends + JerseyTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(TestRMWebServicesCapacitySchedDynamicConfig.class); + + protected static MockRM rm; + + private static class WebServletModule extends ServletModule { + private final Configuration conf; + + public WebServletModule(Configuration conf) { + this.conf = conf; + } + + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, + YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + serve("/*").with(GuiceContainer.class); + } + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + } + + private void initResourceManager(Configuration conf) throws IOException { + GuiceServletConfig.setInjector( + Guice.createInjector(new WebServletModule(conf))); + rm.start(); + //Need to call reinitialize as + //MutableCSConfigurationProvider with InMemoryConfigurationStore + //somehow does not load the queues properly and falls back to default config. + //Therefore CS will think there's only the default queue there. + ((CapacityScheduler)rm.getResourceScheduler()).reinitialize(conf, + rm.getRMContext(), true); + } + + public TestRMWebServicesCapacitySchedDynamicConfig() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testSchedulerResponsePercentageMode() + throws Exception { + Configuration config = CSConfigGenerator + .createPercentageConfig(); + config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + + initResourceManager(config); + JSONObject json = sendRequestToSchedulerEndpoint(); + validateSchedulerInfo(json, "percentage", "root.default", "root.test1", + "root.test2"); + } + + @Test + public void testSchedulerResponseAbsoluteMode() + throws Exception { + Configuration config = CSConfigGenerator + .createAbsoluteConfig(); + config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + + initResourceManager(config); + JSONObject json = sendRequestToSchedulerEndpoint(); + validateSchedulerInfo(json, "absolute", "root.default", "root.test1", + "root.test2"); + } + + @Test + public void testSchedulerResponseWeightMode() + throws Exception { + Configuration config = CSConfigGenerator + .createWeightConfig(); + config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + + initResourceManager(config); + JSONObject json = sendRequestToSchedulerEndpoint(); + validateSchedulerInfo(json, "weight", "root.default", "root.test1", + "root.test2"); + } + + private JSONObject sendRequestToSchedulerEndpoint() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + String jsonString = response.getEntity(String.class); + LOG.debug("Received JSON response: " + jsonString); + return new JSONObject(jsonString); + } + + private void validateSchedulerInfo(JSONObject json, String expectedMode, + String... expectedQueues) throws JSONException { + int expectedQSize = expectedQueues.length; + Assert.assertNotNull("SchedulerTypeInfo should not be null", json); + assertEquals("incorrect number of elements in: " + json, 1, json.length()); + + JSONObject info = json.getJSONObject("scheduler"); + Assert.assertNotNull("Scheduler object should not be null", json); + assertEquals("incorrect number of elements in: " + info, 1, info.length()); + + //Validate if root queue has the expected mode + info = info.getJSONObject("schedulerInfo"); + Assert.assertNotNull("SchedulerInfo should not be null", info); + Assert.assertEquals("Expected Queue mode " +expectedMode, expectedMode, + info.getString("mode")); + + JSONObject queuesObj = info.getJSONObject("queues"); + Assert.assertNotNull("QueueInfoList should not be null", queuesObj); + + JSONArray queueArray = queuesObj.getJSONArray("queue"); + Assert.assertNotNull("Queue list should not be null", queueArray); + assertEquals("QueueInfoList should be size of " + expectedQSize, + expectedQSize, queueArray.length()); + + // Create mapping of queue path -> mode + Map modesMap = new HashMap<>(); + for (int i = 0; i < queueArray.length(); i++) { + JSONObject obj = queueArray.getJSONObject(i); + String queuePath = CapacitySchedulerConfiguration.ROOT + "." + + obj.getString("queueName"); + String mode = obj.getString("mode"); + modesMap.put(queuePath, mode); + } + + //Validate queue paths and modes + List sortedExpectedPaths = Arrays.stream(expectedQueues) + .sorted(Comparator.comparing(String::toLowerCase)) + .collect(Collectors.toList()); + + List sortedActualPaths = modesMap.keySet().stream() + .sorted(Comparator.comparing(String::toLowerCase)) + .collect(Collectors.toList()); + Assert.assertEquals("Expected Queue paths: " + sortedExpectedPaths, + sortedExpectedPaths, sortedActualPaths); + + // Validate if we have a single "mode" for all queues + Set modesSet = new HashSet<>(modesMap.values()); + Assert.assertEquals("Expected a single Queue mode for all queues: " + + expectedMode + ", got: " + modesMap, 1, modesSet.size()); + Assert.assertEquals("Expected Queue mode " + expectedMode, + expectedMode, modesSet.iterator().next()); + } + + private static class CSConfigGenerator { + public static Configuration createPercentageConfig() { + Map conf = new HashMap<>(); + conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2"); + conf.put("yarn.scheduler.capacity.root.test1.capacity", "50"); + conf.put("yarn.scheduler.capacity.root.test2.capacity", "50"); + conf.put("yarn.scheduler.capacity.root.test1.maximum-capacity", "100"); + conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING"); + conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING"); + return createConfiguration(conf); + } + + public static Configuration createAbsoluteConfig() { + Map conf = new HashMap<>(); + conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2"); + conf.put("yarn.scheduler.capacity.root.capacity", + "[memory=6136,vcores=30]"); + conf.put("yarn.scheduler.capacity.root.default.capacity", + "[memory=3064,vcores=15]"); + conf.put("yarn.scheduler.capacity.root.test1.capacity", + "[memory=2048,vcores=10]"); + conf.put("yarn.scheduler.capacity.root.test2.capacity", + "[memory=1024,vcores=5]"); + conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING"); + conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING"); + return createConfiguration(conf); + } + + public static Configuration createWeightConfig() { + Map conf = new HashMap<>(); + conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2"); + conf.put("yarn.scheduler.capacity.root.capacity", "1w"); + conf.put("yarn.scheduler.capacity.root.default.capacity", "10w"); + conf.put("yarn.scheduler.capacity.root.test1.capacity", "4w"); + conf.put("yarn.scheduler.capacity.root.test2.capacity", "6w"); + conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING"); + conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING"); + return createConfiguration(conf); + } + + public static Configuration createConfiguration( + Map configs) { + Configuration config = new Configuration(); + + for (Map.Entry entry: configs.entrySet()) { + config.set(entry.getKey(), entry.getValue()); + } + return config; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index eb7677f80a..5ce943e5f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -574,7 +574,7 @@ private void verifySchedulerInfoJson(JSONObject json) JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 12, info.length()); + assertEquals("incorrect number of elements", 13, info.length()); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONArray partitionsCapsArray = capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);