big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10139. ValidateAndGetSchedulerConfiguration API fails when cluster max allocation > default 8GB. Contributed by Prabhu Joseph.

Browse Source
This commit is contained in:
Sunil G 2020-02-19 11:17:22 +05:30
parent ac4b556e2d
commit 6526f95bd2
2 changed files with 37 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -31,6 +31,7 @@
import java.util.EnumSet; import java.util.EnumSet;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
@ -2639,14 +2640,22 @@ public synchronized Response validateAndGetSchedulerConfiguration(
((MutableConfScheduler) scheduler).getMutableConfProvider(); ((MutableConfScheduler) scheduler).getMutableConfProvider();
Configuration schedulerConf = mutableConfigurationProvider Configuration schedulerConf = mutableConfigurationProvider
.getConfiguration(); .getConfiguration();
Configuration newConfig = mutableConfigurationProvider Configuration newSchedulerConf = mutableConfigurationProvider
.applyChanges(schedulerConf, mutationInfo); .applyChanges(schedulerConf, mutationInfo);
Configuration yarnConf = ((CapacityScheduler) scheduler).getConf(); Configuration yarnConf = ((CapacityScheduler) scheduler).getConf();
Configuration newConfig = new Configuration(yarnConf);
Iterator<Map.Entry<String, String>> iter = newSchedulerConf.iterator();
Entry<String, String> e = null;
while (iter.hasNext()) {
e = iter.next();
newConfig.set(e.getKey(), e.getValue());
}
CapacitySchedulerConfigValidator.validateCSConfiguration(yarnConf, CapacitySchedulerConfigValidator.validateCSConfiguration(yarnConf,
newConfig, rm.getRMContext()); newConfig, rm.getRMContext());
return Response.status(Status.OK) return Response.status(Status.OK)
.entity(new ConfInfo(newConfig)) .entity(new ConfInfo(newSchedulerConf))
.build(); .build();
} catch (Exception e) { } catch (Exception e) {
String errorMsg = "CapacityScheduler configuration validation failed:" String errorMsg = "CapacityScheduler configuration validation failed:"

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
View File

@ -763,6 +763,32 @@ public void testGlobalConfChange() throws Exception {
newCSConf.getMaximumSystemApplications()); newCSConf.getMaximumSystemApplications());
} }
@Test
public void testValidateWithClusterMaxAllocation() throws Exception {
WebResource r = resource();
int clusterMax = YarnConfiguration.
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB * 2;
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
clusterMax);
SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> updateParam = new HashMap<>();
updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_APPLICATIONS_SUFFIX,
"100");
QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
ClientResponse response =
r.path("ws").path("v1").path("cluster")
.path(RMWSConsts.SCHEDULER_CONF_VALIDATE)
.queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.post(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
}
@Override @Override
@After @After
public void tearDown() throws Exception { public void tearDown() throws Exception {