big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6866. Minor clean-up and fixes in anticipation of YARN-2915 merge with trunk. (Botong Huang via Subru).

Browse Source 
(cherry picked from commit 40453879ec860819b080ccc82454480a436c0adc)
This commit is contained in:
Subru Krishnan 2017-07-25 20:22:45 -07:00 committed by Carlo Curino
parent 8737c2a8e9
commit 86f05c3312
5 changed files with 16 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-project/pom.xml
View File

@ -99,9 +99,9 @@
<ldap-api.version>1.0.0-M33</ldap-api.version>
<jcache.version>1.0.0</jcache.version>
<ehcache.version>3.0.3</ehcache.version>
<hikari.version>2.4.11</hikari.version>
<mssql.version>6.1.0.jre7</mssql.version>
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>2.4.12</hikari.version>
<mssql.version>6.2.1.jre7</mssql.version>
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -2645,7 +2645,7 @@ public static boolean isAclEnabled(Configuration conf) {
ROUTER_PREFIX + "clientrm.";
public static final String ROUTER_CLIENTRM_ADDRESS =
ROUTER_CLIENTRM_PREFIX + ".address";
ROUTER_CLIENTRM_PREFIX + "address";
public static final int DEFAULT_ROUTER_CLIENTRM_PORT = 8050;
public static final String DEFAULT_ROUTER_CLIENTRM_ADDRESS =
"0.0.0.0:" + DEFAULT_ROUTER_CLIENTRM_PORT;
@ -2663,7 +2663,7 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String ROUTER_RMADMIN_PREFIX = ROUTER_PREFIX + "rmadmin.";
public static final String ROUTER_RMADMIN_ADDRESS =
ROUTER_RMADMIN_PREFIX + ".address";
ROUTER_RMADMIN_PREFIX + "address";
public static final int DEFAULT_ROUTER_RMADMIN_PORT = 8052;
public static final String DEFAULT_ROUTER_RMADMIN_ADDRESS =
"0.0.0.0:" + DEFAULT_ROUTER_RMADMIN_PORT;

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
View File

@ -77,6 +77,10 @@ public void initializeMemberVariables() {
.add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
configurationPropsToSkipCompare
.add(YarnConfiguration.RM_EPOCH);
configurationPropsToSkipCompare
.add(YarnConfiguration.ROUTER_CLIENTRM_ADDRESS);
configurationPropsToSkipCompare
.add(YarnConfiguration.ROUTER_RMADMIN_ADDRESS);
// Federation policies configs to be ignored
configurationPropsToSkipCompare

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
View File

@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.router.clientrm;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -88,10 +87,10 @@
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
import org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
@ -175,14 +174,8 @@ protected ApplicationClientProtocol getClientRMProxyForSubCluster(
ApplicationClientProtocol clientRMProxy = null;
try {
clientRMProxy =
user.doAs(new PrivilegedExceptionAction<ApplicationClientProtocol>() {
@Override
public ApplicationClientProtocol run() throws Exception {
return ClientRMProxy.createRMProxy(getConf(),
ApplicationClientProtocol.class);
}
});
clientRMProxy = FederationProxyProviderUtil.createRMProxy(getConf(),
ApplicationClientProtocol.class, subClusterId, user);
} catch (Exception e) {
RouterServerUtil.logAndThrowException(
"Unable to create the interface to reach the SubCluster "

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
View File

@ -209,9 +209,9 @@ Optional:
| Property | Example | Description |
|:---- |:---- |
|`yarn.router.hostname` | `0.0.0.0` | Router host name.
|`yarn.router.clientrm.address` | `0.0.0.0:8032` | Router client address. |
|`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. |
|`yarn.router.webapp.address` | `0.0.0.0:80` | Webapp address at the router. |
|`yarn.router.admin.address` | `0.0.0.0:8033` | Admin address at the router. |
|`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. |
|`yarn.router.webapp.https.address` | `0.0.0.0:443` | Secure webapp address at the router. |
|`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. |
|`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. |
@ -227,7 +227,7 @@ These are extra configurations that should appear in the **conf/yarn-site.xml**
|:---- |:---- |
| `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled.
|`yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor.
| `yarn.client.failover.proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.|
| `yarn.client.failover-proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.|
Optional:
@ -242,7 +242,7 @@ Currently, the only supported implementation of the state-store is Microsoft SQL
Running a Sample Job
--------------------
In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which said jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
| Property | Example | Description |
|:--- |:--- |