big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8559. Expose mutable-conf scheduler's configuration in RM /scheduler-conf endpoint. Contributed by Weiwei Yang.

Browse Source
This commit is contained in:
Weiwei Yang 2018-08-09 23:46:53 +08:00
parent cd04e954d2
commit d352f167eb
6 changed files with 198 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
@ -58,6 +59,12 @@ void logAndApplyMutation(UserGroupInformation user, SchedConfUpdateInfo
*/
void confirmPendingMutation(boolean isValid) throws Exception;
/**
* Returns scheduler configuration cached in this provider.
* @return scheduler configuration.
*/
Configuration getConfiguration();
/**
* Closes the configuration provider, releasing any required resources.
* @throws IOException on failure to close

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
View File

@ -125,6 +125,11 @@ public CapacitySchedulerConfiguration loadConfiguration(Configuration
return new CapacitySchedulerConfiguration(loadedConf, false);
}
@Override
public Configuration getConfiguration() {
return new Configuration(schedConf);
}
@Override
public ConfigurationMutationACLPolicy getAclMutationPolicy() {
return aclMutationPolicy;

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -188,6 +188,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webapp.WebServices;
@ -2372,6 +2373,39 @@ public Void run() throws Exception {
}
}
@GET
@Path(RMWSConsts.SCHEDULER_CONF)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public Response getSchedulerConfiguration(@Context HttpServletRequest hsr)
throws AuthorizationException {
// Only admin user is allowed to read scheduler conf,
// in order to avoid leaking sensitive info, such as ACLs
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
initForWritableEndpoints(callerUGI, true);
ResourceScheduler scheduler = rm.getResourceScheduler();
if (scheduler instanceof MutableConfScheduler
&& ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
MutableConfigurationProvider mutableConfigurationProvider =
((MutableConfScheduler) scheduler).getMutableConfProvider();
// We load the cached configuration from configuration store,
// this should be the conf properties used by the scheduler.
Configuration schedulerConf = mutableConfigurationProvider
.getConfiguration();
return Response.status(Status.OK)
.entity(new ConfInfo(schedulerConf))
.build();
} else {
return Response.status(Status.BAD_REQUEST).entity(
"This API only supports to retrieve scheduler configuration"
+ " from a mutable-conf scheduler, underneath scheduler "
+ scheduler.getClass().getSimpleName()
+ " is not an instance of MutableConfScheduler")
.build();
}
}
@GET
@Path(RMWSConsts.CHECK_USER_ACCESS_TO_QUEUE)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,

72
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java Normal file
View File

@ -0,0 +1,72 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import org.apache.hadoop.conf.Configuration;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
@XmlRootElement(name = "configuration")
@XmlAccessorType(XmlAccessType.FIELD)
public class ConfInfo {
protected ArrayList<ConfItem> property = new ArrayList<>();
public ConfInfo() {
} // JAXB needs this
public ConfInfo(Configuration conf) {
conf.forEach(entry ->
add(new ConfItem(entry.getKey(), entry.getValue())));
}
public void add(ConfItem confItem) {
property.add(confItem);
}
public ArrayList<ConfItem> getItems() {
return property;
}
@XmlAccessorType(XmlAccessType.FIELD)
public static class ConfItem {
private String name;
private String value;
public ConfItem() {
// JAXB needs this
}
public ConfItem(String name, String value){
this.name = name;
this.value = value;
}
public String getKey() {
return name;
}
public String getValue() {
return value;
}
}
}

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
View File

@ -39,6 +39,9 @@
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -56,6 +59,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
/**
* Test scheduler configuration mutation via REST API.
@ -157,8 +161,40 @@ public TestRMWebServicesConfigurationMutation() {
.contextPath("jersey-guice-filter").servletPath("/").build());
}
private CapacitySchedulerConfiguration getSchedulerConf()
throws JSONException {
WebResource r = resource();
ClientResponse response =
r.path("ws").path("v1").path("cluster")
.queryParam("user.name", userName).path("scheduler-conf")
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
JSONObject json = response.getEntity(JSONObject.class);
JSONArray items = (JSONArray) json.get("property");
CapacitySchedulerConfiguration parsedConf =
new CapacitySchedulerConfiguration();
for (int i=0; i<items.length(); i++) {
JSONObject obj = (JSONObject) items.get(i);
parsedConf.set(obj.get("name").toString(),
obj.get("value").toString());
}
return parsedConf;
}
@Test
public void testGetSchedulerConf() throws Exception {
CapacitySchedulerConfiguration orgConf = getSchedulerConf();
assertNotNull(orgConf);
assertEquals(3, orgConf.getQueues("root").length);
}
@Test
public void testAddNestedQueue() throws Exception {
CapacitySchedulerConfiguration orgConf = getSchedulerConf();
assertNotNull(orgConf);
assertEquals(3, orgConf.getQueues("root").length);
WebResource r = resource();
ClientResponse response;
@ -198,6 +234,10 @@ public void testAddNestedQueue() throws Exception {
0.01f);
assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"),
0.01f);
CapacitySchedulerConfiguration newConf = getSchedulerConf();
assertNotNull(newConf);
assertEquals(4, newConf.getQueues("root").length);
}
@Test

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
View File

@ -4964,6 +4964,7 @@ Please note that this feature is currently in the alpha stage and is subject to
### HTTP Operations Supported
* GET
* PUT
### Elements of the *sched-conf* object
@ -4975,6 +4976,45 @@ Please note that this feature is currently in the alpha stage and is subject to
| remove-queue | string | Full path name of a queue to remove |
| global-updates | map | Map of key value pairs to update scheduler's global configuration |
### GET Request Examples
Get requests are used to retrieve the scheduler's configuration that is currently loaded into scheduler's context.
**XML response**
HTTP Request:
Accept: application/xml
Content-Type: application/xml
GET http://rm-http-address:port/ws/v1/cluster/scheduler-conf
Response Header:
TTP/1.1 200 OK
Content-Type: application/xml; charset=utf-8
Transfer-Encoding: chunked
Response Body:
```xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<configuration>
<property>
<name>yarn.scheduler.capacity.root.queues</name>
<value>default</value>
</property>
<property>
<name>yarn.scheduler.capacity.maximum-applications</name>
<value>10000</value>
</property>
<property>
<name>yarn.scheduler.capacity.root.default.capacity</name>
<value>100</value>
</property>
</configuration>
```
### PUT Request Examples
Put requests are used to modify the scheduler configuration. A successful mutation results in a 200 response. A malformed request or one which resulted in an invalid scheduler configuration results in a 400 response.