big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3948. Display Application Priority in RM Web UI.(Sunil G via rohithsharmaks)

Browse Source
This commit is contained in:
Rohith Sharma K S 2015-08-07 10:43:41 +05:30
parent 6d4eee718a
commit b6265d39c5
15 changed files with 116 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -159,6 +159,8 @@ Release 2.8.0 - UNRELEASED
YARN-3736. Add RMStateStore apis to store and load accepted reservations for
failover (adhoot via asuresh)
YARN-3948. Display Application Priority in RM Web UI.(Sunil G via rohithsharmaks)
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
View File

@ -408,4 +408,17 @@ public abstract void setLogAggregationStatus(
@Public
@Unstable
public abstract void setUnmanagedApp(boolean unmanagedApplication);
/**
* Get priority of the application
*
* @return Application's priority
*/
@Public
@Stable
public abstract Priority getPriority();
@Private
@Unstable
public abstract void setPriority(Priority priority);
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -196,6 +196,7 @@ message ApplicationReportProto {
repeated string applicationTags = 20;
optional LogAggregationStatusProto log_aggregation_status = 21;
optional bool unmanaged_application = 22 [default = false];
optional PriorityProto priority = 23;
}
enum LogAggregationStatusProto {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
View File

@ -501,6 +501,8 @@ private int printApplicationReport(String applicationId)
appReportStr.println(appReport.getUser());
appReportStr.print("\tQueue : ");
appReportStr.println(appReport.getQueue());
appReportStr.print("\tApplication Priority : ");
appReportStr.println(appReport.getPriority());
appReportStr.print("\tStart-Time : ");
appReportStr.println(appReport.getStartTime());
appReportStr.print("\tFinish-Time : ");

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -106,6 +106,7 @@ public void testGetApplicationReport() throws Exception {
FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
null, null, false);
newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
newApplicationReport.setPriority(Priority.newInstance(0));
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
@ -119,6 +120,7 @@ public void testGetApplicationReport() throws Exception {
pw.println("\tApplication-Type : YARN");
pw.println("\tUser : user");
pw.println("\tQueue : queue");
pw.println("\tApplication Priority : 0");
pw.println("\tStart-Time : 0");
pw.println("\tFinish-Time : 0");
pw.println("\tProgress : 53.79%");

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
@ -36,6 +37,7 @@
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.TextFormat;
@ -55,6 +57,7 @@ public class ApplicationReportPBImpl extends ApplicationReport {
private Token clientToAMToken = null;
private Token amRmToken = null;
private Set<String> applicationTags = null;
private Priority priority = null;
public ApplicationReportPBImpl() {
builder = ApplicationReportProto.newBuilder();
@ -484,6 +487,11 @@ private void mergeLocalToBuilder() {
builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
if (this.priority != null
&& !((PriorityPBImpl) this.priority).getProto().equals(
builder.getPriority())) {
builder.setPriority(convertToProtoFormat(this.priority));
}
}
private void mergeLocalToProto() {
@ -551,6 +559,14 @@ private TokenProto convertToProtoFormat(Token t) {
return ((TokenPBImpl)t).getProto();
}
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
return new PriorityPBImpl(p);
}
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl)t).getProto();
}
@Override
public LogAggregationStatus getLogAggregationStatus() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
@ -593,4 +609,25 @@ public void setUnmanagedApp(boolean unmanagedApplication) {
maybeInitBuilder();
builder.setUnmanagedApplication(unmanagedApplication);
}
@Override
public Priority getPriority() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
if (this.priority != null) {
return this.priority;
}
if (!p.hasPriority()) {
return null;
}
this.priority = convertFromProtoFormat(p.getPriority());
return this.priority;
}
@Override
public void setPriority(Priority priority) {
maybeInitBuilder();
if (priority == null)
builder.clearPriority();
this.priority = priority;
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
View File

@ -324,7 +324,8 @@ public static ApplicationReport newApplicationReport(
String url, long startTime, long finishTime,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String appType, Token amRmToken, Set<String> tags) {
float progress, String appType, Token amRmToken, Set<String> tags,
Priority priority) {
ApplicationReport report = recordFactory
.newRecordInstance(ApplicationReport.class);
report.setApplicationId(applicationId);
@ -347,6 +348,7 @@ public static ApplicationReport newApplicationReport(
report.setApplicationType(appType);
report.setAMRMToken(amRmToken);
report.setApplicationTags(tags);
report.setPriority(priority);
return report;
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
View File

@ -167,6 +167,7 @@ public ApplicationReport run() throws Exception {
._("Application Type:", app.getType())
._("Application Tags:",
app.getApplicationTags() == null ? "" : app.getApplicationTags())
._("Application Priority:", clarifyAppPriority(app.getPriority()))
._(
"YarnApplicationState:",
app.getAppState() == null ? UNAVAILABLE : clarifyAppState(app
@ -342,6 +343,10 @@ private String clarifyAppState(YarnApplicationState state) {
}
}
private String clarifyAppPriority(int priority) {
return priority + " (Higher Integer value indicates higher priority)";
}
private String clairfyAppFinalStatus(FinalApplicationStatus status) {
if (status == FinalApplicationStatus.UNDEFINED) {
return "Application has not completed yet.";

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
View File

@ -52,13 +52,13 @@ private static String getAppsTableColumnDefs(
.append("{'sType':'string', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }")
.append("\n, {'sType':'numeric', 'aTargets': " +
(isFairSchedulerPage ? "[6, 7]": "[5, 6]"))
(isFairSchedulerPage ? "[6, 7]": "[6, 7]"))
.append(", 'mRender': renderHadoopDate }")
.append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':");
if (isFairSchedulerPage) {
sb.append("[13]");
} else if (isResourceManager) {
sb.append("[12]");
sb.append("[13]");
} else {
sb.append("[9]");
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
View File

@ -58,6 +58,7 @@ public class AppInfo {
protected long finishedTime;
protected long elapsedTime;
protected String applicationTags;
protected int priority;
private int allocatedCpuVcores;
private int allocatedMemoryMB;
protected boolean unmanagedApplication;
@ -86,6 +87,10 @@ public AppInfo(ApplicationReport app) {
finishedTime = app.getFinishTime();
elapsedTime = Times.elapsed(startedTime, finishedTime);
finalAppStatus = app.getFinalApplicationStatus();
priority = 0;
if (app.getPriority() != null) {
priority = app.getPriority().getPriority();
}
if (app.getApplicationResourceUsageReport() != null) {
runningContainers = app.getApplicationResourceUsageReport()
.getNumUsedContainers();
@ -194,4 +199,8 @@ public String getApplicationTags() {
public boolean isUnmanagedApp() {
return unmanagedApplication;
}
public int getPriority() {
return priority;
}
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -660,10 +660,10 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName,
ApplicationReport report = BuilderUtils.newApplicationReport(
this.applicationId, currentApplicationAttemptId, this.user,
this.queue, this.name, host, rpcPort, clientToAMToken,
createApplicationState(), diags,
trackingUrl, this.startTime, this.finishTime, finishState,
appUsageReport, origTrackingUrl, progress, this.applicationType,
amrmToken, applicationTags);
createApplicationState(), diags, trackingUrl, this.startTime,
this.finishTime, finishState, appUsageReport, origTrackingUrl,
progress, this.applicationType, amrmToken, applicationTags,
this.submissionContext.getPriority());
report.setLogAggregationStatus(logAggregationStatus);
report.setUnmanagedApp(submissionContext.getUnmanagedAM());
return report;

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
View File

@ -56,7 +56,8 @@ protected void renderData(Block html) {
TBODY<TABLE<Hamlet>> tbody =
html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
.th(".name", "Name").th(".type", "Application Type")
.th(".queue", "Queue").th(".starttime", "StartTime")
.th(".queue", "Queue").th(".priority", "Application Priority")
.th(".starttime", "StartTime")
.th(".finishtime", "FinishTime").th(".state", "State")
.th(".finalstatus", "FinalStatus")
.th(".runningcontainer", "Running Containers")
@ -106,7 +107,9 @@ protected void renderData(Block html) {
.append("\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
.getQueue()))).append("\",\"").append(app.getStartedTime())
.getQueue()))).append("\",\"").append(String
.valueOf(app.getPriority()))
.append("\",\"").append(app.getStartedTime())
.append("\",\"").append(app.getFinishedTime())
.append("\",\"")
.append(app.getAppState() == null ? UNAVAILABLE : app.getAppState())

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -74,7 +74,8 @@ public class AppInfo {
protected long clusterId;
protected String applicationType;
protected String applicationTags = "";
protected int priority;
// these are only allowed if acls allow
protected long startedTime;
protected long finishedTime;
@ -130,6 +131,11 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
this.user = app.getUser().toString();
this.name = app.getName().toString();
this.queue = app.getQueue().toString();
this.priority = 0;
if (app.getApplicationSubmissionContext().getPriority() != null) {
this.priority = app.getApplicationSubmissionContext().getPriority()
.getPriority();
}
this.progress = app.getProgress() * 100;
this.diagnostics = app.getDiagnostics().toString();
if (diagnostics == null || diagnostics.isEmpty()) {
@ -328,4 +334,8 @@ public LogAggregationStatus getLogAggregationStatus() {
public boolean isUnmanagedApp() {
return unmanagedApplication;
}
public int getPriority() {
return this.priority;
}
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -1290,6 +1290,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException,
WebServicesTestUtils.getXmlString(element, "name"),
WebServicesTestUtils.getXmlString(element, "applicationType"),
WebServicesTestUtils.getXmlString(element, "queue"),
WebServicesTestUtils.getXmlInt(element, "priority"),
WebServicesTestUtils.getXmlString(element, "state"),
WebServicesTestUtils.getXmlString(element, "finalStatus"),
WebServicesTestUtils.getXmlFloat(element, "progress"),
@ -1316,18 +1317,18 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException,
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
assertEquals("incorrect number of elements", 29, info.length());
assertEquals("incorrect number of elements", 30, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("applicationType"),
info.getString("queue"), info.getString("state"),
info.getString("finalStatus"), (float) info.getDouble("progress"),
info.getString("trackingUI"), info.getString("diagnostics"),
info.getLong("clusterId"), info.getLong("startedTime"),
info.getLong("finishedTime"), info.getLong("elapsedTime"),
info.getString("amHostHttpAddress"), info.getString("amContainerLogs"),
info.getInt("allocatedMB"), info.getInt("allocatedVCores"),
info.getInt("runningContainers"),
info.getString("queue"), info.getInt("priority"),
info.getString("state"), info.getString("finalStatus"),
(float) info.getDouble("progress"), info.getString("trackingUI"),
info.getString("diagnostics"), info.getLong("clusterId"),
info.getLong("startedTime"), info.getLong("finishedTime"),
info.getLong("elapsedTime"), info.getString("amHostHttpAddress"),
info.getString("amContainerLogs"), info.getInt("allocatedMB"),
info.getInt("allocatedVCores"), info.getInt("runningContainers"),
info.getInt("preemptedResourceMB"),
info.getInt("preemptedResourceVCores"),
info.getInt("numNonAMContainerPreempted"),
@ -1337,8 +1338,8 @@ public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
}
public void verifyAppInfoGeneric(RMApp app, String id, String user,
String name, String applicationType, String queue, String state,
String finalStatus, float progress, String trackingUI,
String name, String applicationType, String queue, int prioirty,
String state, String finalStatus, float progress, String trackingUI,
String diagnostics, long clusterId, long startedTime, long finishedTime,
long elapsedTime, String amHostHttpAddress, String amContainerLogs,
int allocatedMB, int allocatedVCores, int numContainers,
@ -1355,6 +1356,7 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user,
WebServicesTestUtils.checkStringMatch("applicationType",
app.getApplicationType(), applicationType);
WebServicesTestUtils.checkStringMatch("queue", app.getQueue(), queue);
assertEquals("priority doesn't match", 0, prioirty);
WebServicesTestUtils.checkStringMatch("state", app.getState().toString(),
state);
WebServicesTestUtils.checkStringMatch("finalStatus", app

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
View File

@ -1382,6 +1382,7 @@ Response Body:
"memorySeconds" : 151730,
"vcoreSeconds" : 103,
"unmanagedApplication":"false"
"applicationPriority":0
},
{
"finishedTime" : 1326815789546,
@ -1408,6 +1409,7 @@ Response Body:
"memorySeconds" : 640064,
"vcoreSeconds" : 442
"unmanagedApplication":"false"
"applicationPriority":0
}
]
}
@ -1458,6 +1460,7 @@ Response Body:
<memorySeconds>151730</memorySeconds>
<vcoreSeconds>103</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
<app>
<id>application_1326815542473_0002</id>
@ -1484,6 +1487,7 @@ Response Body:
<memorySeconds>640064</memorySeconds>
<vcoreSeconds>442</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
</apps>
```
@ -1644,6 +1648,7 @@ Note that depending on security settings a user might not be able to see all the
| memorySeconds | long | The amount of memory the application has allocated (megabyte-seconds) |
| vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) |
| unmanagedApplication | boolean | Is the application unmanaged. |
| applicationPriority | int | priority of the submitted application |
### Response Examples
@ -1685,6 +1690,7 @@ Response Body:
"memorySeconds" : 151730,
"vcoreSeconds" : 103,
"unmanagedApplication":"false"
"applicationPriority":0
}
}
```
@ -1727,6 +1733,7 @@ Response Body:
<memorySeconds>151730</memorySeconds>
<vcoreSeconds>103</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
```