big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6871. Add additional deSelects params in RMWebServices#getAppReport. Contributed by Tanuj Nayak.

Browse Source
This commit is contained in:
Sunil G 2017-09-27 14:37:32 +05:30
parent dd378775df
commit 8facf1f976
3 changed files with 162 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
View File

@ -91,7 +91,16 @@ public enum DeSelectType {
* <code>RESOURCE_REQUESTS</code> is the first
* supported type from YARN-6280.
*/
RESOURCE_REQUESTS("resourceRequests");
RESOURCE_REQUESTS("resourceRequests"),
/**
* <code>APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION,
* RESOURCE_INFO</code> are additionally supported parameters added in
* YARN-6871.
*/
TIMEOUTS("timeouts"),
APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"),
AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"),
RESOURCE_INFO("resourceInfo");
private final String literals;

112
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -123,7 +123,7 @@ public class AppInfo {
protected String amNodeLabelExpression;
protected ResourcesInfo resourceInfo = null;
protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo();
private AppTimeoutsInfo timeouts;
public AppInfo() {
} // JAXB needs this
@ -247,47 +247,87 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
unmanagedApplication = appSubmissionContext.getUnmanagedAM();
appNodeLabelExpression =
app.getApplicationSubmissionContext().getNodeLabelExpression();
amNodeLabelExpression = (unmanagedApplication) ? null
: app.getAMResourceRequests().get(0).getNodeLabelExpression();
/*
* When the deSelects parameter contains "amNodeLabelExpression", objects
* pertaining to the amNodeLabelExpression are not returned. By default,
* this is not skipped. (YARN-6871)
*/
if(!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
amNodeLabelExpression = (unmanagedApplication) ?
null :
app.getAMResourceRequests().get(0).getNodeLabelExpression();
}
/*
* When the deSelects parameter contains "appNodeLabelExpression", objects
* pertaining to the appNodeLabelExpression are not returned. By default,
* this is not skipped. (YARN-6871)
*/
if (!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) {
appNodeLabelExpression =
app.getApplicationSubmissionContext().getNodeLabelExpression();
}
/*
* When the deSelects parameter contains "amNodeLabelExpression", objects
* pertaining to the amNodeLabelExpression are not returned. By default,
* this is not skipped. (YARN-6871)
*/
if (!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
amNodeLabelExpression = (unmanagedApplication) ?
null :
app.getAMResourceRequests().get(0).getNodeLabelExpression();
}
/*
* When the deSelects parameter contains "resourceInfo", ResourceInfo
* objects are not returned. Default behavior is no skipping. (YARN-6871)
*/
// Setting partition based resource usage of application
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
if (scheduler instanceof CapacityScheduler) {
RMAppAttempt attempt = app.getCurrentAppAttempt();
if (null != attempt) {
FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
.getApplicationAttempt(attempt.getAppAttemptId());
resourceInfo = null != ficaAppAttempt
? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage())
: null;
}
}
Map<ApplicationTimeoutType, Long> applicationTimeouts =
app.getApplicationTimeouts();
if (applicationTimeouts.isEmpty()) {
// If application is not set timeout, lifetime should be sent as default
// with expiryTime=UNLIMITED and remainingTime=-1
AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
timeouts.add(timeoutInfo);
} else {
for (Map.Entry<ApplicationTimeoutType, Long> entry : app
.getApplicationTimeouts().entrySet()) {
AppTimeoutInfo timeout = new AppTimeoutInfo();
timeout.setTimeoutType(entry.getKey());
long timeoutInMillis = entry.getValue().longValue();
timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
if (app.isAppInCompletedStates()) {
timeout.setRemainingTime(0);
} else {
timeout.setRemainingTime(Math
.max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
if (!deSelects.contains(DeSelectType.RESOURCE_INFO)) {
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
if (scheduler instanceof CapacityScheduler) {
RMAppAttempt attempt = app.getCurrentAppAttempt();
if (null != attempt) {
FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
.getApplicationAttempt(attempt.getAppAttemptId());
resourceInfo = null != ficaAppAttempt ?
new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) :
null;
}
timeouts.add(timeout);
}
}
/*
* When the deSelects parameter contains "appTimeouts", objects pertaining
* to app timeouts are not returned. By default, this is not skipped.
* (YARN-6871)
*/
if (!deSelects.contains(DeSelectType.TIMEOUTS)) {
Map<ApplicationTimeoutType, Long> applicationTimeouts =
app.getApplicationTimeouts();
if (applicationTimeouts.isEmpty()) {
// If application is not set timeout, lifetime should be sent
// as default with expiryTime=UNLIMITED and remainingTime=-1
AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
timeouts = new AppTimeoutsInfo();
timeouts.add(timeoutInfo);
} else {
for (Map.Entry<ApplicationTimeoutType, Long> entry : app
.getApplicationTimeouts().entrySet()) {
AppTimeoutInfo timeout = new AppTimeoutInfo();
timeout.setTimeoutType(entry.getKey());
long timeoutInMillis = entry.getValue().longValue();
timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
if (app.isAppInCompletedStates()) {
timeout.setRemainingTime(0);
} else {
timeout.setRemainingTime(Math.max(
(timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
}
timeouts.add(timeout);
}
}
}
}
}

77
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -1113,8 +1113,83 @@ public void testAppsQueryWithDeselects()
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
JSONObject app = array.getJSONObject(0);
assertTrue("resource requests shouldn't exits",
assertTrue("resource requests shouldn't exist",
!app.has("resourceRequests"));
params.clear();
params.add("deSelects",
DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString());
response =
r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
app = array.getJSONObject(0);
assertTrue("AMNodeLabelExpression shouldn't exist",
!app.has("amNodeLabelExpression"));
params.clear();
params.add("deSelects", DeSelectFields.DeSelectType.TIMEOUTS.toString());
response =
r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
app = array.getJSONObject(0);
assertTrue("Timeouts shouldn't exist", !app.has("timeouts"));
rm.stop();
params.clear();
params.add("deSelects",
DeSelectFields.DeSelectType.APP_NODE_LABEL_EXPRESSION.toString());
response =
r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
app = array.getJSONObject(0);
assertTrue("AppNodeLabelExpression shouldn't exist",
!app.has("appNodeLabelExpression"));
rm.stop();
params.clear();
params
.add("deSelects", DeSelectFields.DeSelectType.RESOURCE_INFO.toString());
response =
r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
app = array.getJSONObject(0);
assertTrue("Resource info shouldn't exist", !app.has("resourceInfo"));
rm.stop();
}