From c60a68ce1d2b7823f78e942cb00d48975476a5a6 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 22 Jul 2016 09:16:38 -0700 Subject: [PATCH] HADOOP-13392. [Azure Data Lake] OAuth2 configuration should be default set to true when AdlFileSystem is configured. Contributed by Vishwajeet Dusane and Chris Douglas. --- .../web/PrivateAzureDataLakeFileSystem.java | 8 +++ .../hdfs/web/TestConfigurationSetting.java | 58 ++++++++++++++----- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/PrivateAzureDataLakeFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/PrivateAzureDataLakeFileSystem.java index c4a19d5abd..aa7c5524e7 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/PrivateAzureDataLakeFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/PrivateAzureDataLakeFileSystem.java @@ -54,6 +54,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.VersionInfo; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_DEFAULT; import java.io.IOException; import java.io.InputStream; @@ -108,6 +110,12 @@ public PrivateAzureDataLakeFileSystem() { @Override public synchronized void initialize(URI uri, Configuration conf) throws IOException { + if (!conf.getBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, + DFS_WEBHDFS_OAUTH_ENABLED_DEFAULT)) { + // clone configuration, enable OAuth2 + conf = new Configuration(conf); + conf.setBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, true); + } super.initialize(uri, conf); overrideOwner = getConf() .getBoolean(ADLConfKeys.ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER, diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java index 9a54ec25aa..9f33fb4150 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java @@ -19,18 +19,23 @@ package org.apache.hadoop.hdfs.web; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.adl.TestableAdlFileSystem; -import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; -import org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider; -import org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider; -import org.junit.Assert; -import org.junit.Test; - import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.adl.TestableAdlFileSystem; +import org.apache.hadoop.hdfs.web.oauth2.AccessTokenProvider; +import org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.ACCESS_TOKEN_PROVIDER_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY; +import static org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider.OAUTH_CREDENTIAL_KEY; + +import org.junit.Assert; +import org.junit.Test; + /** * This class is responsible for testing adl file system required configuration * and feature set keys. @@ -41,14 +46,13 @@ public class TestConfigurationSetting { public void testAllConfiguration() throws URISyntaxException, IOException { TestableAdlFileSystem fs = new TestableAdlFileSystem(); Configuration conf = new Configuration(); - conf.set(HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY, - "http://localhost:1111/refresh"); - conf.set(CredentialBasedAccessTokenProvider.OAUTH_CREDENTIAL_KEY, - "credential"); - conf.set(HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY, "MY_CLIENTID"); - conf.set(HdfsClientConfigKeys.ACCESS_TOKEN_PROVIDER_KEY, - ConfCredentialBasedAccessTokenProvider.class.getName()); - conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY, "true"); + conf.set(OAUTH_REFRESH_URL_KEY, "http://localhost:1111/refresh"); + conf.set(OAUTH_CREDENTIAL_KEY, "credential"); + conf.set(OAUTH_CLIENT_ID_KEY, "MY_CLIENTID"); + conf.setClass(ACCESS_TOKEN_PROVIDER_KEY, + ConfCredentialBasedAccessTokenProvider.class, + AccessTokenProvider.class); + conf.setBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, true); URI uri = new URI("adl://localhost:1234"); fs.initialize(uri, conf); @@ -109,4 +113,26 @@ public void testAllConfiguration() throws URISyntaxException, IOException { Assert.assertEquals(false, fs.isOverrideOwnerFeatureOn()); fs.close(); } + + @Test + public void testOAuthEnable() throws Exception { + try (TestableAdlFileSystem fs = new TestableAdlFileSystem()) { + Configuration conf = new Configuration(); + conf.set(OAUTH_REFRESH_URL_KEY, "http://localhost:1111/refresh"); + conf.set(OAUTH_CREDENTIAL_KEY, "credential"); + conf.set(OAUTH_CLIENT_ID_KEY, "MY_CLIENTID"); + conf.setClass(ACCESS_TOKEN_PROVIDER_KEY, + ConfCredentialBasedAccessTokenProvider.class, + AccessTokenProvider.class); + // disable OAuth2 in configuration, verify overridden + conf.setBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, false); + + URI uri = new URI("adl://localhost:1234"); + fs.initialize(uri, conf); + Assert.assertFalse(conf.getBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, false)); + Assert.assertTrue(fs.getConf().getBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, + false)); + } + } + }