big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13392. [Azure Data Lake] OAuth2 configuration should be default set to true when AdlFileSystem is configured. Contributed by Vishwajeet Dusane and Chris Douglas.

Browse Source
This commit is contained in:
Chris Nauroth 2016-07-22 09:16:38 -07:00
parent dca6335dbc
commit c60a68ce1d
2 changed files with 50 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/PrivateAzureDataLakeFileSystem.java
View File

@ -54,6 +54,8 @@
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.util.VersionInfo;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_DEFAULT;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@ -108,6 +110,12 @@ public PrivateAzureDataLakeFileSystem() {
@Override @Override
public synchronized void initialize(URI uri, Configuration conf) public synchronized void initialize(URI uri, Configuration conf)
throws IOException { throws IOException {
if (!conf.getBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY,
DFS_WEBHDFS_OAUTH_ENABLED_DEFAULT)) {
// clone configuration, enable OAuth2
conf = new Configuration(conf);
conf.setBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, true);
}
super.initialize(uri, conf); super.initialize(uri, conf);
overrideOwner = getConf() overrideOwner = getConf()
.getBoolean(ADLConfKeys.ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER, .getBoolean(ADLConfKeys.ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER,

58
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java
View File

@ -19,18 +19,23 @@
package org.apache.hadoop.hdfs.web; package org.apache.hadoop.hdfs.web;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.adl.TestableAdlFileSystem;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider;
import org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.adl.TestableAdlFileSystem;
import org.apache.hadoop.hdfs.web.oauth2.AccessTokenProvider;
import org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.ACCESS_TOKEN_PROVIDER_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
import static org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider.OAUTH_CREDENTIAL_KEY;
import org.junit.Assert;
import org.junit.Test;
/** /**
* This class is responsible for testing adl file system required configuration * This class is responsible for testing adl file system required configuration
* and feature set keys. * and feature set keys.
@ -41,14 +46,13 @@ public class TestConfigurationSetting {
public void testAllConfiguration() throws URISyntaxException, IOException { public void testAllConfiguration() throws URISyntaxException, IOException {
TestableAdlFileSystem fs = new TestableAdlFileSystem(); TestableAdlFileSystem fs = new TestableAdlFileSystem();
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set(HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY, conf.set(OAUTH_REFRESH_URL_KEY, "http://localhost:1111/refresh");
"http://localhost:1111/refresh"); conf.set(OAUTH_CREDENTIAL_KEY, "credential");
conf.set(CredentialBasedAccessTokenProvider.OAUTH_CREDENTIAL_KEY, conf.set(OAUTH_CLIENT_ID_KEY, "MY_CLIENTID");
"credential"); conf.setClass(ACCESS_TOKEN_PROVIDER_KEY,
conf.set(HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY, "MY_CLIENTID"); ConfCredentialBasedAccessTokenProvider.class,
conf.set(HdfsClientConfigKeys.ACCESS_TOKEN_PROVIDER_KEY, AccessTokenProvider.class);
ConfCredentialBasedAccessTokenProvider.class.getName()); conf.setBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, true);
conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY, "true");
URI uri = new URI("adl://localhost:1234"); URI uri = new URI("adl://localhost:1234");
fs.initialize(uri, conf); fs.initialize(uri, conf);
@ -109,4 +113,26 @@ public void testAllConfiguration() throws URISyntaxException, IOException {
Assert.assertEquals(false, fs.isOverrideOwnerFeatureOn()); Assert.assertEquals(false, fs.isOverrideOwnerFeatureOn());
fs.close(); fs.close();
} }
@Test
public void testOAuthEnable() throws Exception {
try (TestableAdlFileSystem fs = new TestableAdlFileSystem()) {
Configuration conf = new Configuration();
conf.set(OAUTH_REFRESH_URL_KEY, "http://localhost:1111/refresh");
conf.set(OAUTH_CREDENTIAL_KEY, "credential");
conf.set(OAUTH_CLIENT_ID_KEY, "MY_CLIENTID");
conf.setClass(ACCESS_TOKEN_PROVIDER_KEY,
ConfCredentialBasedAccessTokenProvider.class,
AccessTokenProvider.class);
// disable OAuth2 in configuration, verify overridden
conf.setBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, false);
URI uri = new URI("adl://localhost:1234");
fs.initialize(uri, conf);
Assert.assertFalse(conf.getBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, false));
Assert.assertTrue(fs.getConf().getBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY,
false));
}
}
} }