YARN-10339. Fix TimelineClient in NodeManager failing when Simple Http Auth used in Secure Cluster

Contributed by Tarun Parimi.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.ha.ActiveStandbyElector;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.util.BasicDiskValidator;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -3266,6 +3267,18 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final long DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS
       = 2000L;
 
+  /**
+   * The configuration prefix of timeline HTTP authentication.
+   */
+  public static final String TIMELINE_HTTP_AUTH_PREFIX =
+          TIMELINE_SERVICE_PREFIX + "http-authentication.";
+
+  /**
+   * The authentication type for timeline HTTP authentication.
+   */
+  public static final String TIMELINE_HTTP_AUTH_TYPE =
+          TIMELINE_HTTP_AUTH_PREFIX + AuthenticationFilter.AUTH_TYPE;
+
   // mark app-history related configs @Private as application history is going
   // to be integrated into the timeline service
   @Private

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java

@@ -43,6 +43,7 @@
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
@@ -315,7 +316,9 @@ public YarnClientApplication createApplication()
 
     // Automatically add the timeline DT into the CLC
     // Only when the security and the timeline service are both enabled
-    if (isSecurityEnabled() && timelineV1ServiceEnabled) {
+    if (isSecurityEnabled() && timelineV1ServiceEnabled &&
+            getConfig().get(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE)
+                    .equals(KerberosAuthenticationHandler.TYPE)) {
       addTimelineDelegationToken(appContext.getAMContainerSpec());
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java

@@ -21,6 +21,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
@@ -1228,6 +1229,9 @@ private void testCreateTimelineClientWithError(
         timelineClientBestEffort);
     conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION,
         timelineVersion);
+    // Timeline Delegation token and client is only used for kerberos
+    conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE,
+            KerberosAuthenticationHandler.TYPE);
     MockYarnClient client = new MockYarnClient();
     MockYarnClient spyClient = spy(client);
     when(spyClient.createTimelineClient()).thenThrow(mockErr);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
@@ -128,7 +129,8 @@ public void testBestEffortTimelineDelegationToken()
     Configuration conf = getConf();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
-
+    conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE,
+            KerberosAuthenticationHandler.TYPE);
     YarnClientImpl client = spy(new YarnClientImpl() {
 
       @Override
@@ -278,6 +280,8 @@ public void testAutomaticTimelineDelegationTokenLoading()
     Configuration conf = getConf();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
+    conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE,
+            KerberosAuthenticationHandler.TYPE);
     TimelineDelegationTokenIdentifier timelineDT =
             new TimelineDelegationTokenIdentifier();
     final Token<TimelineDelegationTokenIdentifier> dToken =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java

@@ -28,6 +28,7 @@
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -349,7 +350,9 @@ private static void putTimelineDataInJSONFile(String path, String type) {
     client.start();
     try {
       if (UserGroupInformation.isSecurityEnabled()
-          && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
+          && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)
+          && conf.get(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE)
+              .equals(KerberosAuthenticationHandler.TYPE)) {
         Token<TimelineDelegationTokenIdentifier> token =
             client.getDelegationToken(
                 UserGroupInformation.getCurrentUser().getUserName());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java

@@ -34,6 +34,8 @@
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLSocketFactory;
 
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -111,8 +113,12 @@ protected void serviceInit(Configuration conf) throws Exception {
     } else {
       connConfigurator = DEFAULT_TIMEOUT_CONN_CONFIGURATOR;
     }
-
+    String defaultAuth = UserGroupInformation.isSecurityEnabled() ?
-    if (UserGroupInformation.isSecurityEnabled()) {
+            KerberosAuthenticationHandler.TYPE :
+            PseudoAuthenticationHandler.TYPE;
+    String authType = conf.get(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE,
+            defaultAuth);
+    if (authType.equals(KerberosAuthenticationHandler.TYPE)) {
       authenticator = new KerberosDelegationTokenAuthenticator();
     } else {
       authenticator = new PseudoDelegationTokenAuthenticator();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java

@@ -243,6 +243,7 @@ public void testDelegationTokenOperationsRetry() throws Exception {
     // use kerberos to bypass the issue in HADOOP-11215
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
+    conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE, "kerberos");
     UserGroupInformation.setConfiguration(conf);
 
     TimelineClientImpl client = createTimelineClient(conf);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java

@@ -48,6 +48,7 @@
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
 import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -107,11 +108,11 @@ public static void setup() {
     try {
       testTimelineServer = new ApplicationHistoryServer();
       conf = new Configuration(false);
-      conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type",
+      conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type",
           "kerberos");
-      conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
+      conf.set(TIMELINE_HTTP_AUTH_PREFIX +
           KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal);
-      conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
+      conf.set(TIMELINE_HTTP_AUTH_PREFIX +
           KerberosAuthenticationHandler.KEYTAB,
           httpSpnegoKeytabFile.getAbsolutePath());
       conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java

@@ -31,6 +31,7 @@
 import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
 import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -48,12 +49,6 @@
  */
 public class TimelineAuthenticationFilterInitializer extends FilterInitializer {
 
-  /**
-   * The configuration prefix of timeline HTTP authentication.
-   */
-  public static final String PREFIX =
-      "yarn.timeline-service.http-authentication.";
-
   @VisibleForTesting
   Map<String, String> filterConfig;
 
@@ -68,7 +63,8 @@ protected void setAuthFilterConfig(Configuration conf) {
     // yarn.timeline-service.http-authentication.proxyuser will override
     // hadoop.proxyuser
     Map<String, String> timelineAuthProps =
-        AuthenticationFilterInitializer.getFilterConfigMap(conf, PREFIX);
+        AuthenticationFilterInitializer.getFilterConfigMap(conf,
+                TIMELINE_HTTP_AUTH_PREFIX);
 
     filterConfig.putAll(timelineAuthProps);
   }
@@ -81,7 +77,8 @@ protected Map<String, String> getFilterConfig() {
    * Initializes {@link TimelineAuthenticationFilter}.
    * <p>
    * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN
-   * configuration properties prefixed with {@value #PREFIX}.
+   * configuration properties prefixed with
+   * {@value org.apache.hadoop.yarn.conf.YarnConfiguration#TIMELINE_HTTP_AUTH_PREFIX}.
    *
    * @param container
    *          The filter container.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java

@@ -23,7 +23,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.FilterContainer;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import static org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer.PREFIX;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
 import org.junit.Test;
 import org.mockito.Mockito;
 
@@ -46,9 +46,9 @@ public void testProxyUserConfiguration() {
         break;
       case 1:
         // yarn.timeline-service.http-authentication.proxyuser prefix
-        conf.set(PREFIX + "proxyuser.foo.hosts", "*");
+        conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.hosts", "*");
-        conf.set(PREFIX + "proxyuser.foo.users", "*");
+        conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.users", "*");
-        conf.set(PREFIX + "proxyuser.foo.groups", "*");
+        conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.groups", "*");
         break;
       case 2:
         // hadoop.proxyuser prefix has been overwritten by
@@ -56,9 +56,9 @@ public void testProxyUserConfiguration() {
         conf.set("hadoop.proxyuser.foo.hosts", "bar");
         conf.set("hadoop.proxyuser.foo.users", "bar");
         conf.set("hadoop.proxyuser.foo.groups", "bar");
-        conf.set(PREFIX + "proxyuser.foo.hosts", "*");
+        conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.hosts", "*");
-        conf.set(PREFIX + "proxyuser.foo.users", "*");
+        conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.users", "*");
-        conf.set(PREFIX + "proxyuser.foo.groups", "*");
+        conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.groups", "*");
         break;
       default:
         break;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java

@@ -68,13 +68,13 @@
 import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
 import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
-import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
 import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
 import org.apache.hadoop.yarn.server.timelineservice.collector.NodeTimelineCollectorManager;
 import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -151,11 +151,11 @@ public static void setup() {
       conf = new Configuration(false);
       conf.setClass("fs.file.impl", RawLocalFileSystem.class,
           FileSystem.class);
-      conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type",
+      conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type",
           "kerberos");
-      conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
+      conf.set(TIMELINE_HTTP_AUTH_PREFIX +
           KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal);
-      conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
+      conf.set(TIMELINE_HTTP_AUTH_PREFIX +
           KerberosAuthenticationHandler.KEYTAB,
           httpSpnegoKeytabFile.getAbsolutePath());
       conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,