diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 2b9b25ac09..6e56f3dd80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; @@ -58,6 +59,12 @@ void logAndApplyMutation(UserGroupInformation user, SchedConfUpdateInfo */ void confirmPendingMutation(boolean isValid) throws Exception; + /** + * Returns scheduler configuration cached in this provider. + * @return scheduler configuration. + */ + Configuration getConfiguration(); + /** * Closes the configuration provider, releasing any required resources. * @throws IOException on failure to close diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 9c3bf9def1..51de437066 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -125,6 +125,11 @@ public CapacitySchedulerConfiguration loadConfiguration(Configuration return new CapacitySchedulerConfiguration(loadedConf, false); } + @Override + public Configuration getConfiguration() { + return new Configuration(schedConf); + } + @Override public ConfigurationMutationACLPolicy getAclMutationPolicy() { return aclMutationPolicy; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 7752fa2be3..01173768c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -188,6 +188,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webapp.WebServices; @@ -2372,6 +2373,39 @@ public Void run() throws Exception { } } + @GET + @Path(RMWSConsts.SCHEDULER_CONF) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public Response getSchedulerConfiguration(@Context HttpServletRequest hsr) + throws AuthorizationException { + // Only admin user is allowed to read scheduler conf, + // in order to avoid leaking sensitive info, such as ACLs + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + initForWritableEndpoints(callerUGI, true); + + ResourceScheduler scheduler = rm.getResourceScheduler(); + if (scheduler instanceof MutableConfScheduler + && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + MutableConfigurationProvider mutableConfigurationProvider = + ((MutableConfScheduler) scheduler).getMutableConfProvider(); + // We load the cached configuration from configuration store, + // this should be the conf properties used by the scheduler. + Configuration schedulerConf = mutableConfigurationProvider + .getConfiguration(); + return Response.status(Status.OK) + .entity(new ConfInfo(schedulerConf)) + .build(); + } else { + return Response.status(Status.BAD_REQUEST).entity( + "This API only supports to retrieve scheduler configuration" + + " from a mutable-conf scheduler, underneath scheduler " + + scheduler.getClass().getSimpleName() + + " is not an instance of MutableConfScheduler") + .build(); + } + } + @GET @Path(RMWSConsts.CHECK_USER_ACCESS_TO_QUEUE) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java new file mode 100644 index 0000000000..bfc450b793 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.hadoop.conf.Configuration; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +@XmlRootElement(name = "configuration") +@XmlAccessorType(XmlAccessType.FIELD) +public class ConfInfo { + + protected ArrayList property = new ArrayList<>(); + + public ConfInfo() { + } // JAXB needs this + + public ConfInfo(Configuration conf) { + conf.forEach(entry -> + add(new ConfItem(entry.getKey(), entry.getValue()))); + } + + public void add(ConfItem confItem) { + property.add(confItem); + } + + public ArrayList getItems() { + return property; + } + + @XmlAccessorType(XmlAccessType.FIELD) + public static class ConfItem { + + private String name; + private String value; + + public ConfItem() { + // JAXB needs this + } + + public ConfItem(String name, String value){ + this.name = name; + this.value = value; + } + + public String getKey() { + return name; + } + + public String getValue() { + return value; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 99b5648cdd..3e2542c7de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -39,6 +39,9 @@ import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -56,6 +59,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; /** * Test scheduler configuration mutation via REST API. @@ -157,8 +161,40 @@ public TestRMWebServicesConfigurationMutation() { .contextPath("jersey-guice-filter").servletPath("/").build()); } + private CapacitySchedulerConfiguration getSchedulerConf() + throws JSONException { + WebResource r = resource(); + ClientResponse response = + r.path("ws").path("v1").path("cluster") + .queryParam("user.name", userName).path("scheduler-conf") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + JSONObject json = response.getEntity(JSONObject.class); + JSONArray items = (JSONArray) json.get("property"); + CapacitySchedulerConfiguration parsedConf = + new CapacitySchedulerConfiguration(); + for (int i=0; i + + + yarn.scheduler.capacity.root.queues + default + + + yarn.scheduler.capacity.maximum-applications + 10000 + + + yarn.scheduler.capacity.root.default.capacity + 100 + + +``` + ### PUT Request Examples Put requests are used to modify the scheduler configuration. A successful mutation results in a 200 response. A malformed request or one which resulted in an invalid scheduler configuration results in a 400 response.