From d0a22bae9bd2c701c9d80896934fe07388cf5ae0 Mon Sep 17 00:00:00 2001 From: Jian He Date: Mon, 28 Dec 2015 15:52:45 -0800 Subject: [PATCH] YARN-4417. Make RM and Timeline-server REST APIs more consistent. Contributed by Wangda Tan --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/webapp/RMWebServices.java | 80 ++++++++++--------- .../webapp/dao/AppAttemptInfo.java | 8 ++ .../webapp/TestRMWebServicesApps.java | 2 +- 4 files changed, 54 insertions(+), 39 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e380893e95..b09823adb0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -76,6 +76,9 @@ Release 2.9.0 - UNRELEASED YARN-4341. add doc about timeline performance tool usage (Chang Li via sjlee) + YARN-4417. Make RM and Timeline-server REST APIs more consistent. + (wtan via jianhe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index b7447651c6..e107537df4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -169,6 +169,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.server.webapp.WebServices; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.AdHocLogDumper; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; @@ -182,7 +185,7 @@ @Singleton @Path("/ws/v1/cluster") -public class RMWebServices { +public class RMWebServices extends WebServices { private static final Log LOG = LogFactory.getLog(RMWebServices.class.getName()); private static final String EMPTY = ""; @@ -201,6 +204,7 @@ public class RMWebServices { @Inject public RMWebServices(final ResourceManager rm, Configuration conf) { + super(rm.getClientRMService()); this.rm = rm; this.conf = conf; isCentralizedNodeLabelConfiguration = @@ -335,8 +339,8 @@ public NodesInfo getNodes(@QueryParam("states") String states) { } } - Collection rmNodes = RMServerUtils.queryRMNodes(this.rm.getRMContext(), - acceptedStates); + Collection rmNodes = RMServerUtils.queryRMNodes( + this.rm.getRMContext(), acceptedStates); NodesInfo nodesInfo = new NodesInfo(); for (RMNode rmNode : rmNodes) { NodeInfo nodeInfo = new NodeInfo(rmNode, sched); @@ -617,39 +621,6 @@ public ApplicationStatisticsInfo getAppStatistics( return appStatInfo; } - private static Set parseQueries( - Set queries, boolean isState) { - Set params = new HashSet(); - if (!queries.isEmpty()) { - for (String query : queries) { - if (query != null && !query.trim().isEmpty()) { - String[] paramStrs = query.split(","); - for (String paramStr : paramStrs) { - if (paramStr != null && !paramStr.trim().isEmpty()) { - if (isState) { - try { - // enum string is in the uppercase - YarnApplicationState.valueOf( - StringUtils.toUpperCase(paramStr.trim())); - } catch (RuntimeException e) { - YarnApplicationState[] stateArray = - YarnApplicationState.values(); - String allAppStates = Arrays.toString(stateArray); - throw new BadRequestException( - "Invalid application-state " + paramStr.trim() - + " specified. It should be one of " + allAppStates); - } - } - params.add( - StringUtils.toLowerCase(paramStr.trim())); - } - } - } - } - } - return params; - } - private static Map> buildScoreboard( Set states, Set types) { Map> scoreboard @@ -728,6 +699,40 @@ public AppAttemptsInfo getAppAttempts(@Context HttpServletRequest hsr, return appAttemptsInfo; } + @GET + @Path("/apps/{appid}/appattempts/{appattemptid}") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Override + public org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo getAppAttempt(@Context HttpServletRequest req, + @Context HttpServletResponse res, @PathParam("appid") String appId, + @PathParam("appattemptid") String appAttemptId) { + init(res); + return super.getAppAttempt(req, res, appId, appAttemptId); + } + + @GET + @Path("/apps/{appid}/appattempts/{appattemptid}/containers") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Override + public ContainersInfo getContainers(@Context HttpServletRequest req, + @Context HttpServletResponse res, @PathParam("appid") String appId, + @PathParam("appattemptid") String appAttemptId) { + init(res); + return super.getContainers(req, res, appId, appAttemptId); + } + + @GET + @Path("/apps/{appid}/appattempts/{appattemptid}/containers/{containerid}") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Override + public ContainerInfo getContainer(@Context HttpServletRequest req, + @Context HttpServletResponse res, @PathParam("appid") String appId, + @PathParam("appattemptid") String appAttemptId, + @PathParam("containerid") String containerId) { + init(res); + return super.getContainer(req, res, appId, appAttemptId, containerId); + } + @GET @Path("/apps/{appid}/state") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) @@ -885,8 +890,7 @@ public Response replaceLabelsOnNode( NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId); Map> newLabelsForNode = new HashMap>(); - newLabelsForNode.put(nid, - new HashSet(newNodeLabelsName)); + newLabelsForNode.put(nid, new HashSet(newNodeLabelsName)); return replaceLabelsOnNode(newLabelsForNode, hsr, "/nodes/nodeid/replace-labels"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index b6e95a6bb3..f465258627 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -36,11 +36,13 @@ public class AppAttemptInfo { protected int id; protected long startTime; + protected long finishedTime; protected String containerId; protected String nodeHttpAddress; protected String nodeId; protected String logsLink; protected String blacklistedNodes; + protected String appAttemptId; public AppAttemptInfo() { } @@ -56,6 +58,7 @@ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user, if (attempt != null) { this.id = attempt.getAppAttemptId().getAttemptId(); this.startTime = attempt.getStartTime(); + this.finishedTime = attempt.getFinishTime(); Container masterContainer = attempt.getMasterContainer(); if (masterContainer != null) { this.containerId = masterContainer.getId().toString(); @@ -75,6 +78,7 @@ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user, } } } + this.appAttemptId = attempt.getAppAttemptId().toString(); } } @@ -86,6 +90,10 @@ public long getStartTime() { return this.startTime; } + public long getFinishedTime() { + return this.finishedTime; + } + public String getNodeHttpAddress() { return this.nodeHttpAddress; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index d0a8c27e52..05f141fa28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1643,7 +1643,7 @@ public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt, String user) throws JSONException, Exception { - assertEquals("incorrect number of elements", 7, info.length()); + assertEquals("incorrect number of elements", 9, info.length()); verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"), info.getLong("startTime"), info.getString("containerId"),