HDFS-12052. Set SWEBHDFS delegation token kind when ssl is enabled in HttpFS. Contributed by Zoran Dimitrijevic.

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java

@@ -95,8 +95,12 @@ protected Properties getConfiguration(String configPrefix,
       throw new RuntimeException("Could not read HttpFS signature secret file: " + signatureSecretFile);
     }
     setAuthHandlerClass(props);
+    String dtkind = WebHdfsConstants.WEBHDFS_TOKEN_KIND.toString();
+    if (conf.getBoolean(HttpFSServerWebServer.SSL_ENABLED_KEY, false)) {
+      dtkind = WebHdfsConstants.SWEBHDFS_TOKEN_KIND.toString();
+    }
     props.setProperty(KerberosDelegationTokenAuthenticationHandler.TOKEN_KIND,
-        WebHdfsConstants.WEBHDFS_TOKEN_KIND.toString());
+                      dtkind);
     return props;
   }
 

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebServer.java

@@ -52,7 +52,7 @@ public class HttpFSServerWebServer {
   private static final String HTTP_HOST_DEFAULT = "0.0.0.0";
 
   // SSL properties
-  private static final String SSL_ENABLED_KEY = "hadoop.httpfs.ssl.enabled";
+  static final String SSL_ENABLED_KEY = "httpfs.ssl.enabled";
   private static final boolean SSL_ENABLED_DEFAULT = false;
 
   private static final String HTTP_ADMINS_KEY =

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java

@@ -28,7 +28,6 @@ public class HttpFSKerberosAuthenticationHandlerForTesting
   @Override
   public void init(Properties config) throws ServletException {
     //NOP overwrite to avoid Kerberos initialization
-    config.setProperty(TOKEN_KIND, "t");
     initTokenManager(config);
   }
 

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java

@@ -20,6 +20,7 @@
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
 import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
 import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
 import org.json.simple.JSONArray;
@@ -46,12 +47,14 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.XAttrCodec;
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
 import org.apache.hadoop.lib.server.Service;
 import org.apache.hadoop.lib.server.ServiceException;
 import org.apache.hadoop.lib.service.Groups;
 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
 import org.apache.hadoop.security.authentication.server.AuthenticationToken;
 import org.apache.hadoop.security.authentication.util.Signer;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.test.HFSTestCase;
 import org.apache.hadoop.test.HadoopUsersConfTestHelper;
 import org.apache.hadoop.test.TestDir;
@@ -120,8 +123,8 @@ public List<String> getGroups(String user) throws IOException {
 
   }
 
-  private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
+  private Configuration createHttpFSConf(boolean addDelegationTokenAuthHandler,
-    throws Exception {
+                                         boolean sslEnabled) throws Exception {
     File homeDir = TestDirHelper.getTestDir();
     Assert.assertTrue(new File(homeDir, "conf").mkdir());
     Assert.assertTrue(new File(homeDir, "log").mkdir());
@@ -133,10 +136,11 @@ private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
     w.write("secret");
     w.close();
 
-    //HDFS configuration
+    // HDFS configuration
     File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf");
     hadoopConfDir.mkdirs();
     Configuration hdfsConf = TestHdfsHelper.getHdfsConf();
+
     // Http Server's conf should be based on HDFS's conf
     Configuration conf = new Configuration(hdfsConf);
     conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
@@ -146,26 +150,39 @@ private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
     conf.writeXml(os);
     os.close();
 
-    //HTTPFS configuration
+    // HTTPFS configuration
     conf = new Configuration(false);
     if (addDelegationTokenAuthHandler) {
-     conf.set("httpfs.authentication.type",
+      conf.set("httpfs.authentication.type",
-              HttpFSKerberosAuthenticationHandlerForTesting.class.getName());
+               HttpFSKerberosAuthenticationHandlerForTesting.class.getName());
     }
     conf.set("httpfs.services.ext", MockGroups.class.getName());
     conf.set("httpfs.admin.group", HadoopUsersConfTestHelper.
-      getHadoopUserGroups(HadoopUsersConfTestHelper.getHadoopUsers()[0])[0]);
+        getHadoopUserGroups(HadoopUsersConfTestHelper.getHadoopUsers()[0])[0]);
-    conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups",
+    conf.set("httpfs.proxyuser." +
+             HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups",
              HadoopUsersConfTestHelper.getHadoopProxyUserGroups());
-    conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts",
+    conf.set("httpfs.proxyuser." +
+             HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts",
              HadoopUsersConfTestHelper.getHadoopProxyUserHosts());
-    conf.set("httpfs.authentication.signature.secret.file", secretFile.getAbsolutePath());
+    conf.set("httpfs.authentication.signature.secret.file",
+             secretFile.getAbsolutePath());
     conf.set("httpfs.hadoop.config.dir", hadoopConfDir.toString());
+    if (sslEnabled) {
+      conf.set("httpfs.ssl.enabled", "true");
+    }
     File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml");
     os = new FileOutputStream(httpfsSite);
     conf.writeXml(os);
     os.close();
+    return conf;
+  }
 
+  private void createHttpFSServer(boolean addDelegationTokenAuthHandler,
+                                  boolean sslEnabled)
+      throws Exception {
+    Configuration conf = createHttpFSConf(addDelegationTokenAuthHandler,
+                                          sslEnabled);
     ClassLoader cl = Thread.currentThread().getContextClassLoader();
     URL url = cl.getResource("webapp");
     WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs");
@@ -177,12 +194,110 @@ private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
     }
   }
 
+  private String getSignedTokenString()
+      throws Exception {
+    AuthenticationToken token = new AuthenticationToken("u", "p",
+        new KerberosDelegationTokenAuthenticationHandler().getType());
+    token.setExpires(System.currentTimeMillis() + 100000000);
+    SignerSecretProvider secretProvider =
+        StringSignerSecretProviderCreator.newStringSignerSecretProvider();
+    Properties secretProviderProps = new Properties();
+    secretProviderProps.setProperty(
+        AuthenticationFilter.SIGNATURE_SECRET, "secret");
+    secretProvider.init(secretProviderProps, null, -1);
+    Signer signer = new Signer(secretProvider);
+    return signer.sign(token.toString());
+  }
+
+  private void delegationTokenCommonTests(boolean sslEnabled) throws Exception {
+    URL url = new URL(TestJettyHelper.getJettyURL(),
+                      "/webhdfs/v1/?op=GETHOMEDIRECTORY");
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
+                        conn.getResponseCode());
+
+    String tokenSigned = getSignedTokenString();
+
+    url = new URL(TestJettyHelper.getJettyURL(),
+                  "/webhdfs/v1/?op=GETDELEGATIONTOKEN");
+    conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestProperty("Cookie",
+                            AuthenticatedURL.AUTH_COOKIE  + "=" + tokenSigned);
+    Assert.assertEquals(HttpURLConnection.HTTP_OK,
+                        conn.getResponseCode());
+
+    JSONObject json = (JSONObject)new JSONParser().parse(
+        new InputStreamReader(conn.getInputStream()));
+    json = (JSONObject)
+      json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_JSON);
+    String tokenStr = (String)json.get(
+        DelegationTokenAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON);
+
+    Token<AbstractDelegationTokenIdentifier> dToken =
+        new Token<AbstractDelegationTokenIdentifier>();
+    dToken.decodeFromUrlString(tokenStr);
+    Assert.assertEquals(sslEnabled ?
+        WebHdfsConstants.SWEBHDFS_TOKEN_KIND :
+        WebHdfsConstants.WEBHDFS_TOKEN_KIND,
+        dToken.getKind());
+
+    url = new URL(TestJettyHelper.getJettyURL(),
+                  "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
+    conn = (HttpURLConnection) url.openConnection();
+    Assert.assertEquals(HttpURLConnection.HTTP_OK,
+                        conn.getResponseCode());
+
+    url = new URL(TestJettyHelper.getJettyURL(),
+                  "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
+    conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod("PUT");
+    Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
+                        conn.getResponseCode());
+
+    url = new URL(TestJettyHelper.getJettyURL(),
+                  "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
+    conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod("PUT");
+    conn.setRequestProperty("Cookie",
+                            AuthenticatedURL.AUTH_COOKIE  + "=" + tokenSigned);
+    Assert.assertEquals(HttpURLConnection.HTTP_OK,
+                        conn.getResponseCode());
+
+    url = new URL(TestJettyHelper.getJettyURL(),
+                  "/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + tokenStr);
+    conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod("PUT");
+    Assert.assertEquals(HttpURLConnection.HTTP_OK,
+                        conn.getResponseCode());
+
+    url = new URL(TestJettyHelper.getJettyURL(),
+                  "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
+    conn = (HttpURLConnection) url.openConnection();
+    Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
+                        conn.getResponseCode());
+
+    // getTrash test with delegation
+    url = new URL(TestJettyHelper.getJettyURL(),
+        "/webhdfs/v1/?op=GETTRASHROOT&delegation=" + tokenStr);
+    conn = (HttpURLConnection) url.openConnection();
+    Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
+        conn.getResponseCode());
+
+    url = new URL(TestJettyHelper.getJettyURL(),
+        "/webhdfs/v1/?op=GETTRASHROOT");
+    conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestProperty("Cookie",
+        AuthenticatedURL.AUTH_COOKIE  + "=" + tokenSigned);
+    Assert.assertEquals(HttpURLConnection.HTTP_OK,
+        conn.getResponseCode());
+  }
+
   @Test
   @TestDir
   @TestJetty
   @TestHdfs
   public void instrumentation() throws Exception {
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     URL url = new URL(TestJettyHelper.getJettyURL(),
                       MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", "nobody"));
@@ -211,7 +326,7 @@ public void instrumentation() throws Exception {
   @TestJetty
   @TestHdfs
   public void testHdfsAccess() throws Exception {
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
     URL url = new URL(TestJettyHelper.getJettyURL(),
@@ -228,7 +343,7 @@ public void testHdfsAccess() throws Exception {
   @TestJetty
   @TestHdfs
   public void testMkdirs() throws Exception {
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
     URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format(
@@ -246,7 +361,7 @@ public void testMkdirs() throws Exception {
   @TestJetty
   @TestHdfs
   public void testGlobFilter() throws Exception {
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
     fs.mkdirs(new Path("/tmp"));
@@ -438,7 +553,7 @@ private byte[] decodeXAttrValue(String value) throws IOException {
   @TestJetty
   @TestHdfs
   public void testPerms() throws Exception {
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
     fs.mkdirs(new Path("/perm"));
@@ -475,7 +590,7 @@ public void testXAttrs() throws Exception {
     final String dir = "/xattrTest";
     final String path = dir + "/file";
 
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
     fs.mkdirs(new Path(dir));
@@ -544,7 +659,7 @@ public void testFileAcls() throws Exception {
     String statusJson;
     List<String> aclEntries;
 
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
     fs.mkdirs(new Path(dir));
@@ -627,7 +742,7 @@ public void testDirAcls() throws Exception {
     String statusJson;
     List<String> aclEntries;
 
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
     fs.mkdirs(new Path(dir));
@@ -665,7 +780,7 @@ public void testDirAcls() throws Exception {
   @TestJetty
   @TestHdfs
   public void testOpenOffsetLength() throws Exception {
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     byte[] array = new byte[]{0, 1, 2, 3};
     FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
@@ -690,7 +805,7 @@ public void testOpenOffsetLength() throws Exception {
   @TestJetty
   @TestHdfs
   public void testPutNoOperation() throws Exception {
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
 
     String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
     URL url = new URL(TestJettyHelper.getJettyURL(),
@@ -708,7 +823,7 @@ public void testPutNoOperation() throws Exception {
   @TestHdfs
   public void testGetTrashRoot() throws Exception {
     String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
-    createHttpFSServer(false);
+    createHttpFSServer(false, false);
     String trashJson = getStatus("/", "GETTRASHROOT");
     String trashPath = getPath(trashJson);
 
@@ -741,99 +856,16 @@ public void testGetTrashRoot() throws Exception {
   @TestJetty
   @TestHdfs
   public void testDelegationTokenOperations() throws Exception {
-    createHttpFSServer(true);
+    createHttpFSServer(true, false);
-
+    delegationTokenCommonTests(false);
-    URL url = new URL(TestJettyHelper.getJettyURL(),
-                      "/webhdfs/v1/?op=GETHOMEDIRECTORY");
-    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-    Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
-                        conn.getResponseCode());
-
-
-    AuthenticationToken token =
-      new AuthenticationToken("u", "p",
-          new KerberosDelegationTokenAuthenticationHandler().getType());
-    token.setExpires(System.currentTimeMillis() + 100000000);
-    SignerSecretProvider secretProvider =
-        StringSignerSecretProviderCreator.newStringSignerSecretProvider();
-    Properties secretProviderProps = new Properties();
-    secretProviderProps.setProperty(AuthenticationFilter.SIGNATURE_SECRET, "secret");
-    secretProvider.init(secretProviderProps, null, -1);
-    Signer signer = new Signer(secretProvider);
-    String tokenSigned = signer.sign(token.toString());
-
-    url = new URL(TestJettyHelper.getJettyURL(),
-                  "/webhdfs/v1/?op=GETHOMEDIRECTORY");
-    conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestProperty("Cookie",
-                            AuthenticatedURL.AUTH_COOKIE  + "=" + tokenSigned);
-    Assert.assertEquals(HttpURLConnection.HTTP_OK,
-                        conn.getResponseCode());
-
-    url = new URL(TestJettyHelper.getJettyURL(),
-                  "/webhdfs/v1/?op=GETDELEGATIONTOKEN");
-    conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestProperty("Cookie",
-                            AuthenticatedURL.AUTH_COOKIE  + "=" + tokenSigned);
-    Assert.assertEquals(HttpURLConnection.HTTP_OK,
-                        conn.getResponseCode());
-
-    JSONObject json = (JSONObject)
-      new JSONParser().parse(new InputStreamReader(conn.getInputStream()));
-    json = (JSONObject)
-      json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_JSON);
-    String tokenStr = (String)
-        json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON);
-
-    url = new URL(TestJettyHelper.getJettyURL(),
-                  "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
-    conn = (HttpURLConnection) url.openConnection();
-    Assert.assertEquals(HttpURLConnection.HTTP_OK,
-                        conn.getResponseCode());
-
-    url = new URL(TestJettyHelper.getJettyURL(),
-                  "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
-    conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestMethod("PUT");
-    Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
-                        conn.getResponseCode());
-
-    url = new URL(TestJettyHelper.getJettyURL(),
-                  "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
-    conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestMethod("PUT");
-    conn.setRequestProperty("Cookie",
-                            AuthenticatedURL.AUTH_COOKIE  + "=" + tokenSigned);
-    Assert.assertEquals(HttpURLConnection.HTTP_OK,
-                        conn.getResponseCode());
-
-    url = new URL(TestJettyHelper.getJettyURL(),
-                  "/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + tokenStr);
-    conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestMethod("PUT");
-    Assert.assertEquals(HttpURLConnection.HTTP_OK,
-                        conn.getResponseCode());
-
-    url = new URL(TestJettyHelper.getJettyURL(),
-                  "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
-    conn = (HttpURLConnection) url.openConnection();
-    Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
-                        conn.getResponseCode());
-
-    // getTrash test with delegation
-    url = new URL(TestJettyHelper.getJettyURL(),
-        "/webhdfs/v1/?op=GETTRASHROOT&delegation=" + tokenStr);
-    conn = (HttpURLConnection) url.openConnection();
-    Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
-        conn.getResponseCode());
-
-    url = new URL(TestJettyHelper.getJettyURL(),
-        "/webhdfs/v1/?op=GETTRASHROOT");
-    conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestProperty("Cookie",
-        AuthenticatedURL.AUTH_COOKIE  + "=" + tokenSigned);
-    Assert.assertEquals(HttpURLConnection.HTTP_OK,
-        conn.getResponseCode());
   }
 
+  @Test
+  @TestDir
+  @TestJetty
+  @TestHdfs
+  public void testDelegationTokenOperationsSsl() throws Exception {
+    createHttpFSServer(true, true);
+    delegationTokenCommonTests(true);
+  }
 }