diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 7301e90abf..61d7c9baaa 100755 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -99,9 +99,9 @@ 1.0.0-M33 1.0.0 - 3.0.3 - 2.4.11 - 6.1.0.jre7 + 3.3.1 + 2.4.12 + 6.2.1.jre7 1.8 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0c0dd113d7..d2013bf6dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2645,7 +2645,7 @@ public static boolean isAclEnabled(Configuration conf) { ROUTER_PREFIX + "clientrm."; public static final String ROUTER_CLIENTRM_ADDRESS = - ROUTER_CLIENTRM_PREFIX + ".address"; + ROUTER_CLIENTRM_PREFIX + "address"; public static final int DEFAULT_ROUTER_CLIENTRM_PORT = 8050; public static final String DEFAULT_ROUTER_CLIENTRM_ADDRESS = "0.0.0.0:" + DEFAULT_ROUTER_CLIENTRM_PORT; @@ -2663,7 +2663,7 @@ public static boolean isAclEnabled(Configuration conf) { public static final String ROUTER_RMADMIN_PREFIX = ROUTER_PREFIX + "rmadmin."; public static final String ROUTER_RMADMIN_ADDRESS = - ROUTER_RMADMIN_PREFIX + ".address"; + ROUTER_RMADMIN_PREFIX + "address"; public static final int DEFAULT_ROUTER_RMADMIN_PORT = 8052; public static final String DEFAULT_ROUTER_RMADMIN_ADDRESS = "0.0.0.0:" + DEFAULT_ROUTER_RMADMIN_PORT; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index a439cd5ac5..a035dc7ebf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -77,6 +77,10 @@ public void initializeMemberVariables() { .add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS); configurationPropsToSkipCompare .add(YarnConfiguration.RM_EPOCH); + configurationPropsToSkipCompare + .add(YarnConfiguration.ROUTER_CLIENTRM_ADDRESS); + configurationPropsToSkipCompare + .add(YarnConfiguration.ROUTER_RMADMIN_ADDRESS); // Federation policies configs to be ignored configurationPropsToSkipCompare diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index ecf53ac10c..7268ebd949 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.router.clientrm; import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -88,10 +87,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; import org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; @@ -175,14 +174,8 @@ protected ApplicationClientProtocol getClientRMProxyForSubCluster( ApplicationClientProtocol clientRMProxy = null; try { - clientRMProxy = - user.doAs(new PrivilegedExceptionAction() { - @Override - public ApplicationClientProtocol run() throws Exception { - return ClientRMProxy.createRMProxy(getConf(), - ApplicationClientProtocol.class); - } - }); + clientRMProxy = FederationProxyProviderUtil.createRMProxy(getConf(), + ApplicationClientProtocol.class, subClusterId, user); } catch (Exception e) { RouterServerUtil.logAndThrowException( "Unable to create the interface to reach the SubCluster " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index 79225b43db..61cb77f3d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -209,9 +209,9 @@ Optional: | Property | Example | Description | |:---- |:---- | |`yarn.router.hostname` | `0.0.0.0` | Router host name. -|`yarn.router.clientrm.address` | `0.0.0.0:8032` | Router client address. | +|`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. | |`yarn.router.webapp.address` | `0.0.0.0:80` | Webapp address at the router. | -|`yarn.router.admin.address` | `0.0.0.0:8033` | Admin address at the router. | +|`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. | |`yarn.router.webapp.https.address` | `0.0.0.0:443` | Secure webapp address at the router. | |`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. | |`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. | @@ -227,7 +227,7 @@ These are extra configurations that should appear in the **conf/yarn-site.xml** |:---- |:---- | | `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled. |`yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor. -| `yarn.client.failover.proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.| +| `yarn.client.failover-proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.| Optional: @@ -242,7 +242,7 @@ Currently, the only supported implementation of the state-store is Microsoft SQL Running a Sample Job -------------------- -In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which said jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations: +In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations: | Property | Example | Description | |:--- |:--- |