big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8529. Add timeout to RouterWebServiceUtil#invokeRMWebService. Contributed by Minni Mittal

Browse Source
This commit is contained in:
bibinchundatt 2021-01-13 15:57:09 +05:30
parent 8ec824f2ba
commit 61f77b7674
7 changed files with 169 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -3956,6 +3956,18 @@ public static boolean isAclEnabled(Configuration conf) {
public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED =
false;
/**
* Connection and Read timeout from the Router to RM.
*/
public static final String ROUTER_WEBAPP_CONNECT_TIMEOUT =
ROUTER_WEBAPP_PREFIX + "connect-timeout";
public static final long DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT =
TimeUnit.SECONDS.toMillis(30);
public static final String ROUTER_WEBAPP_READ_TIMEOUT =
ROUTER_WEBAPP_PREFIX + "read-timeout";
public static final long DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT =
TimeUnit.SECONDS.toMillis(30);
////////////////////////////////
// CSI Volume configs
////////////////////////////////

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
View File

@ -183,6 +183,10 @@ public void initializeMemberVariables() {
.add(YarnConfiguration.ROUTER_CLIENTRM_SUBMIT_RETRY);
configurationPrefixToSkipCompare
.add(YarnConfiguration.ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED);
configurationPrefixToSkipCompare
.add(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT);
configurationPrefixToSkipCompare
.add(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT);
// Set by container-executor.cfg
configurationPrefixToSkipCompare.add(YarnConfiguration.NM_USER_HOME_DIR);

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.router.webapp;
import com.sun.jersey.api.client.Client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -49,10 +50,13 @@ public class AboutBlock extends HtmlBlock {
protected void render(Block html) {
Configuration conf = this.router.getConfig();
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
Client client = RouterWebServiceUtil.createJerseyClient(conf);
ClusterMetricsInfo metrics = RouterWebServiceUtil.genericForward(
webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, conf);
ClusterMetricsInfo metrics = RouterWebServiceUtil
.genericForward(webAppAddress, null, ClusterMetricsInfo.class,
HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
conf, client);
boolean isEnabled = conf.getBoolean(
YarnConfiguration.FEDERATION_ENABLED,
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
View File

@ -24,6 +24,7 @@
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
import com.sun.jersey.api.client.Client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
@ -53,10 +54,12 @@ public class AppsBlock extends HtmlBlock {
protected void render(Block html) {
// Get the applications from the Resource Managers
Configuration conf = this.router.getConfig();
Client client = RouterWebServiceUtil.createJerseyClient(conf);
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
AppsInfo apps = RouterWebServiceUtil.genericForward(webAppAddress, null,
AppsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, conf);
AppsInfo apps = RouterWebServiceUtil
.genericForward(webAppAddress, null, AppsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, conf,
client);
setTitle("Applications");

105
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java
View File

@ -27,6 +27,7 @@
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.Response;
import com.sun.jersey.api.client.Client;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@ -78,6 +79,10 @@ public class DefaultRequestInterceptorREST
private String webAppAddress;
private SubClusterId subClusterId = null;
// It is very expensive to create the client
// Jersey will spawn a thread for every client request
private Client client = null;
public void setWebAppAddress(String webAppAddress) {
this.webAppAddress = webAppAddress;
}
@ -97,6 +102,7 @@ protected SubClusterId getSubClusterId() {
@Override
public void init(String user) {
webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(getConf());
client = RouterWebServiceUtil.createJerseyClient(getConf());
}
@Override
@ -109,7 +115,7 @@ public ClusterInfo getClusterInfo() {
return RouterWebServiceUtil.genericForward(webAppAddress, null,
ClusterInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.INFO, null, null,
getConf());
getConf(), client);
}
@Override
@ -117,7 +123,7 @@ public ClusterUserInfo getClusterUserInfo(HttpServletRequest hsr) {
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
ClusterUserInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.CLUSTER_USER_INFO, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -125,7 +131,7 @@ public ClusterMetricsInfo getClusterMetricsInfo() {
return RouterWebServiceUtil.genericForward(webAppAddress, null,
ClusterMetricsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
getConf());
getConf(), client);
}
@Override
@ -133,7 +139,7 @@ public SchedulerTypeInfo getSchedulerInfo() {
return RouterWebServiceUtil.genericForward(webAppAddress, null,
SchedulerTypeInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER, null, null,
getConf());
getConf(), client);
}
@Override
@ -143,7 +149,7 @@ public String dumpSchedulerLogs(String time, HttpServletRequest hsr)
return RouterWebServiceUtil.genericForward(webAppAddress, null,
String.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_LOGS, null, null,
getConf());
getConf(), client);
}
@Override
@ -156,7 +162,7 @@ public NodesInfo getNodes(String states) {
return RouterWebServiceUtil.genericForward(webAppAddress, null,
NodesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null,
additionalParam, getConf());
additionalParam, getConf(), client);
}
@Override
@ -164,7 +170,7 @@ public NodeInfo getNode(String nodeId) {
return RouterWebServiceUtil.genericForward(webAppAddress, null,
NodeInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -172,9 +178,10 @@ public ResourceInfo updateNodeResource(HttpServletRequest hsr,
String nodeId, ResourceOptionInfo resourceOption) {
final String nodePath =
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId;
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
ResourceInfo.class, HTTPMethods.POST,
nodePath + "/resource", resourceOption, null, getConf());
return RouterWebServiceUtil
.genericForward(webAppAddress, hsr, ResourceInfo.class,
HTTPMethods.POST, nodePath + "/resource", resourceOption, null,
getConf(), client);
}
@Override
@ -187,7 +194,7 @@ public AppsInfo getApps(HttpServletRequest hsr, String stateQuery,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
AppsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null,
getConf());
getConf(), client);
}
@Override
@ -197,7 +204,7 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
ActivitiesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_ACTIVITIES, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -206,7 +213,7 @@ public BulkActivitiesInfo getBulkActivities(HttpServletRequest hsr,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
BulkActivitiesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_BULK_ACTIVITIES,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -218,7 +225,7 @@ public AppActivitiesInfo getAppActivities(HttpServletRequest hsr,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
AppActivitiesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_APP_ACTIVITIES,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -228,7 +235,7 @@ public ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
ApplicationStatisticsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APP_STATISTICS, null, null,
getConf());
getConf(), client);
}
@Override
@ -238,7 +245,7 @@ public AppInfo getApp(HttpServletRequest hsr, String appId,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
AppInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -247,7 +254,7 @@ public AppState getAppState(HttpServletRequest hsr, String appId)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
AppState.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.STATE,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -257,7 +264,7 @@ public Response updateAppState(AppState targetState, HttpServletRequest hsr,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.STATE,
targetState, null, getConf());
targetState, null, getConf(), client);
}
@Override
@ -266,7 +273,7 @@ public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
NodeToLabelsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_NODE_TO_LABELS, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -281,7 +288,7 @@ public LabelsToNodesInfo getLabelsToNodes(Set<String> labels)
return RouterWebServiceUtil.genericForward(webAppAddress, null,
LabelsToNodesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.LABEL_MAPPINGS, null,
additionalParam, getConf());
additionalParam, getConf(), client);
}
@Override
@ -290,7 +297,7 @@ public Response replaceLabelsOnNodes(NodeToLabelsEntryList newNodeToLabels,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.REPLACE_NODE_TO_LABELS,
newNodeToLabels, null, getConf());
newNodeToLabels, null, getConf(), client);
}
@Override
@ -301,7 +308,7 @@ public Response replaceLabelsOnNode(Set<String> newNodeLabelsName,
.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.NODES + "/" + nodeId + "/replace-labels",
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -310,7 +317,7 @@ public NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
NodeLabelsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_NODE_LABELS, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -319,7 +326,7 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.ADD_NODE_LABELS,
newNodeLabels, null, getConf());
newNodeLabels, null, getConf(), client);
}
@Override
@ -329,7 +336,7 @@ public Response removeFromCluserNodeLabels(Set<String> oldNodeLabels,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.REMOVE_NODE_LABELS, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -338,7 +345,7 @@ public NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
NodeLabelsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.NODES + "/" + nodeId + "/get-labels",
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -347,7 +354,7 @@ public AppPriority getAppPriority(HttpServletRequest hsr, String appId)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
AppPriority.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.PRIORITY,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -357,7 +364,7 @@ public Response updateApplicationPriority(AppPriority targetPriority,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.PRIORITY,
targetPriority, null, getConf());
targetPriority, null, getConf(), client);
}
@Override
@ -366,7 +373,7 @@ public AppQueue getAppQueue(HttpServletRequest hsr, String appId)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
AppQueue.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.QUEUE,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -376,7 +383,7 @@ public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.QUEUE,
targetQueue, null, getConf());
targetQueue, null, getConf(), client);
}
@Override
@ -385,7 +392,7 @@ public Response createNewApplication(HttpServletRequest hsr)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS_NEW_APPLICATION, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -395,7 +402,7 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, newApp, null,
getConf());
getConf(), client);
}
@Override
@ -405,7 +412,7 @@ public Response postDelegationToken(DelegationToken tokenData,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN, tokenData,
null, getConf());
null, getConf(), client);
}
@Override
@ -415,7 +422,7 @@ public Response postDelegationTokenExpiration(HttpServletRequest hsr)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN_EXPIRATION,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -425,7 +432,7 @@ public Response cancelDelegationToken(HttpServletRequest hsr)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.DELETE,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -434,7 +441,7 @@ public Response createNewReservation(HttpServletRequest hsr)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_NEW, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -444,7 +451,7 @@ public Response submitReservation(ReservationSubmissionRequestInfo resContext,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_SUBMIT,
resContext, null, getConf());
resContext, null, getConf(), client);
}
@Override
@ -454,7 +461,7 @@ public Response updateReservation(ReservationUpdateRequestInfo resContext,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_UPDATE,
resContext, null, getConf());
resContext, null, getConf(), client);
}
@Override
@ -464,7 +471,7 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_DELETE,
resContext, null, getConf());
resContext, null, getConf(), client);
}
@Override
@ -476,7 +483,7 @@ public Response listReservation(String queue, String reservationId,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_LIST, null,
null, getConf());
null, getConf(), client);
}
@Override
@ -486,7 +493,7 @@ public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId,
.genericForward(webAppAddress, hsr, AppTimeoutInfo.class,
HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS
+ "/" + appId + "/" + RMWSConsts.TIMEOUTS + "/" + type,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -495,7 +502,7 @@ public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId)
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
AppTimeoutsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUTS,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -505,7 +512,7 @@ public Response updateApplicationTimeout(AppTimeoutInfo appTimeout,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUT,
appTimeout, null, getConf());
appTimeout, null, getConf(), client);
}
@Override
@ -513,7 +520,7 @@ public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) {
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
AppAttemptsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH
+ RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -522,7 +529,7 @@ public RMQueueAclInfo checkUserAccessToQueue(String queue, String username,
return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
RMQueueAclInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.QUEUES + "/" + queue
+ "/access", null, null, getConf());
+ "/access", null, null, getConf(), client);
}
@Override
@ -532,7 +539,7 @@ public AppAttemptInfo getAppAttempt(HttpServletRequest req,
AppAttemptInfo.class,
HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/"
+ appId + "/" + RMWSConsts.APPATTEMPTS + "/" + appAttemptId,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -543,7 +550,7 @@ public ContainersInfo getContainers(HttpServletRequest req,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/"
+ RMWSConsts.APPATTEMPTS + "/" + appAttemptId + "/"
+ RMWSConsts.CONTAINERS,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -555,7 +562,7 @@ public ContainerInfo getContainer(HttpServletRequest req,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/"
+ RMWSConsts.APPATTEMPTS + "/" + appAttemptId + "/"
+ RMWSConsts.CONTAINERS + "/" + containerId,
null, null, getConf());
null, null, getConf(), client);
}
@Override
@ -573,6 +580,6 @@ public Response signalToContainer(String containerId, String command,
.genericForward(webAppAddress, req, Response.class, HTTPMethods.POST,
RMWSConsts.RM_WEB_SERVICE_PATH + "/" + RMWSConsts.CONTAINERS + "/"
+ containerId + "/" + RMWSConsts.SIGNAL + "/" + command, null,
null, getConf());
null, getConf(), client);
}
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.router.webapp;
import com.sun.jersey.api.client.Client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
@ -53,10 +54,12 @@ public class NodesBlock extends HtmlBlock {
protected void render(Block html) {
// Get the node info from the federation
Configuration conf = this.router.getConfig();
Client client = RouterWebServiceUtil.createJerseyClient(conf);
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
NodesInfo nodes = RouterWebServiceUtil.genericForward(webAppAddress, null,
NodesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf);
NodesInfo nodes = RouterWebServiceUtil
.genericForward(webAppAddress, null, NodesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf,
client);
setTitle("Nodes");

118
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
View File

@ -30,6 +30,7 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.core.HttpHeaders;
@ -53,7 +54,6 @@
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -92,13 +92,14 @@ private RouterWebServiceUtil() {
* @param formParam the form parameters as input for a specific REST call
* @param additionalParam the query parameters as input for a specific REST
* call in case the call has no servlet request
* @param client same client used to reduce number of clients created
* @return the retrieved entity from the REST call
*/
protected static <T> T genericForward(
final String webApp, final HttpServletRequest hsr,
final Class<T> returnType, final HTTPMethods method,
final String targetPath, final Object formParam,
final Map<String, String[]> additionalParam, Configuration conf) {
protected static <T> T genericForward(final String webApp,
final HttpServletRequest hsr, final Class<T> returnType,
final HTTPMethods method, final String targetPath, final Object formParam,
final Map<String, String[]> additionalParam, Configuration conf,
Client client) {
UserGroupInformation callerUGI = null;
@ -130,26 +131,34 @@ public T run() {
paramMap = additionalParam;
}
ClientResponse response = RouterWebServiceUtil.invokeRMWebService(
webApp, targetPath, method,
(hsr == null) ? null : hsr.getPathInfo(), paramMap, formParam,
getMediaTypeFromHttpServletRequest(hsr, returnType), conf);
ClientResponse response = RouterWebServiceUtil
.invokeRMWebService(webApp, targetPath, method,
(hsr == null) ? null : hsr.getPathInfo(), paramMap, formParam,
getMediaTypeFromHttpServletRequest(hsr, returnType), conf,
client);
if (Response.class.equals(returnType)) {
return (T) RouterWebServiceUtil.clientResponseToResponse(response);
}
// YARN RM can answer with Status.OK or it throws an exception
if (response.getStatus() == SC_OK) {
return response.getEntity(returnType);
}
if (response.getStatus() == SC_NO_CONTENT) {
try {
return returnType.getConstructor().newInstance();
} catch (RuntimeException | ReflectiveOperationException e) {
LOG.error("Cannot create empty entity for {}", returnType, e);
try {
// YARN RM can answer with Status.OK or it throws an exception
if (response.getStatus() == SC_OK) {
return response.getEntity(returnType);
}
if (response.getStatus() == SC_NO_CONTENT) {
try {
return returnType.getConstructor().newInstance();
} catch (RuntimeException | ReflectiveOperationException e) {
LOG.error("Cannot create empty entity for {}", returnType, e);
}
}
RouterWebServiceUtil.retrieveException(response);
return null;
} finally {
if (response != null) {
response.close();
}
}
RouterWebServiceUtil.retrieveException(response);
return null;
}
});
} catch (InterruptedException e) {
@ -161,14 +170,21 @@ public T run() {
/**
* Performs an invocation of a REST call on a remote RMWebService.
*
* @param additionalParam
* @param webApp the address of the remote webap
* @param path to add to the webapp address
* @param method the HTTP method of the REST call
* @param additionalPath the servlet request path
* @param queryParams hsr of additional Param
* @param formParam the form parameters as input for a specific REST call
* @param mediaType Media type for Servlet request call
* @param conf to support http and https
* @param client same client used to reduce number of clients created
* @return Client response to REST call
*/
private static ClientResponse invokeRMWebService(String webApp, String path,
HTTPMethods method, String additionalPath,
Map<String, String[]> queryParams, Object formParam, String mediaType,
Configuration conf) {
Client client = WebServiceClient.getWebServiceClient().createClient();
Configuration conf, Client client) {
InetSocketAddress socketAddress = NetUtils
.getConnectAddress(NetUtils.createSocketAddr(webApp));
String scheme = YarnConfiguration.useHttps(conf) ? "https://" : "http://";
@ -202,21 +218,25 @@ private static ClientResponse invokeRMWebService(String webApp, String path,
ClientResponse response = null;
switch (method) {
case DELETE:
response = builder.delete(ClientResponse.class);
break;
case GET:
response = builder.get(ClientResponse.class);
break;
case POST:
response = builder.post(ClientResponse.class);
break;
case PUT:
response = builder.put(ClientResponse.class);
break;
default:
break;
try {
switch (method) {
case DELETE:
response = builder.delete(ClientResponse.class);
break;
case GET:
response = builder.get(ClientResponse.class);
break;
case POST:
response = builder.post(ClientResponse.class);
break;
case PUT:
response = builder.put(ClientResponse.class);
break;
default:
break;
}
} finally {
client.destroy();
}
return response;
@ -316,6 +336,24 @@ public static AppsInfo mergeAppsInfo(ArrayList<AppInfo> appsInfo,
return allApps;
}
/**
* Create a Jersey client instance.
* @param conf Configuration
* @return a jersey client
*/
protected static Client createJerseyClient(Configuration conf) {
Client client = Client.create();
client.setConnectTimeout((int) conf
.getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT,
TimeUnit.MILLISECONDS));
client.setReadTimeout((int) conf
.getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT,
TimeUnit.MILLISECONDS));
return client;
}
private static AppInfo mergeUAMWithUAM(AppInfo uam1, AppInfo uam2) {
AppInfo partialReport = new AppInfo();
partialReport.setAppId(uam1.getAppId());