YARN-6866. Minor clean-up and fixes in anticipation of YARN-2915 merge with trunk. (Botong Huang via Subru).

(cherry picked from commit 40453879ec860819b080ccc82454480a436c0adc)

hadoop-project/pom.xml

@@ -99,9 +99,9 @@
     <ldap-api.version>1.0.0-M33</ldap-api.version>
 
     <jcache.version>1.0.0</jcache.version>
-    <ehcache.version>3.0.3</ehcache.version>
+    <ehcache.version>3.3.1</ehcache.version>
-    <hikari.version>2.4.11</hikari.version>
+    <hikari.version>2.4.12</hikari.version>
-    <mssql.version>6.1.0.jre7</mssql.version>
+    <mssql.version>6.2.1.jre7</mssql.version>
 
     <!-- define the Java language version used by the compiler -->
     <javac.version>1.8</javac.version>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -2645,7 +2645,7 @@ public static boolean isAclEnabled(Configuration conf) {
       ROUTER_PREFIX + "clientrm.";
 
   public static final String ROUTER_CLIENTRM_ADDRESS =
-      ROUTER_CLIENTRM_PREFIX + ".address";
+      ROUTER_CLIENTRM_PREFIX + "address";
   public static final int DEFAULT_ROUTER_CLIENTRM_PORT = 8050;
   public static final String DEFAULT_ROUTER_CLIENTRM_ADDRESS =
       "0.0.0.0:" + DEFAULT_ROUTER_CLIENTRM_PORT;
@@ -2663,7 +2663,7 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final String ROUTER_RMADMIN_PREFIX = ROUTER_PREFIX + "rmadmin.";
 
   public static final String ROUTER_RMADMIN_ADDRESS =
-      ROUTER_RMADMIN_PREFIX + ".address";
+      ROUTER_RMADMIN_PREFIX + "address";
   public static final int DEFAULT_ROUTER_RMADMIN_PORT = 8052;
   public static final String DEFAULT_ROUTER_RMADMIN_ADDRESS =
       "0.0.0.0:" + DEFAULT_ROUTER_RMADMIN_PORT;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java

@@ -77,6 +77,10 @@ public void initializeMemberVariables() {
         .add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
     configurationPropsToSkipCompare
         .add(YarnConfiguration.RM_EPOCH);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.ROUTER_CLIENTRM_ADDRESS);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.ROUTER_RMADMIN_ADDRESS);
 
     // Federation policies configs to be ignored
     configurationPropsToSkipCompare

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.router.clientrm;
 
 import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -88,10 +87,10 @@
 import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
@@ -175,14 +174,8 @@ protected ApplicationClientProtocol getClientRMProxyForSubCluster(
 
     ApplicationClientProtocol clientRMProxy = null;
     try {
-      clientRMProxy =
+      clientRMProxy = FederationProxyProviderUtil.createRMProxy(getConf(),
-          user.doAs(new PrivilegedExceptionAction<ApplicationClientProtocol>() {
+          ApplicationClientProtocol.class, subClusterId, user);
-            @Override
-            public ApplicationClientProtocol run() throws Exception {
-              return ClientRMProxy.createRMProxy(getConf(),
-                  ApplicationClientProtocol.class);
-            }
-          });
     } catch (Exception e) {
       RouterServerUtil.logAndThrowException(
           "Unable to create the interface to reach the SubCluster "

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md

@@ -209,9 +209,9 @@ Optional:
 | Property | Example | Description |
 |:---- |:---- |
 |`yarn.router.hostname` | `0.0.0.0` | Router host name.
-|`yarn.router.clientrm.address` | `0.0.0.0:8032` | Router client address. |
+|`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. |
 |`yarn.router.webapp.address` | `0.0.0.0:80` | Webapp address at the router. |
-|`yarn.router.admin.address` | `0.0.0.0:8033` | Admin address at the router. |
+|`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. |
 |`yarn.router.webapp.https.address` | `0.0.0.0:443` | Secure webapp address at the router. |
 |`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. |
 |`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. |
@@ -227,7 +227,7 @@ These are extra configurations that should appear in the **conf/yarn-site.xml**
 |:---- |:---- |
 | `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled.
 |`yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor.
-| `yarn.client.failover.proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.|
+| `yarn.client.failover-proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.|
 
 Optional:
 
@@ -242,7 +242,7 @@ Currently, the only supported implementation of the state-store is Microsoft SQL
 
 Running a Sample Job
 --------------------
-In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which said jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
+In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
 
 | Property | Example | Description |
 |:--- |:--- |