HADOOP-13609. Refine credential provider related codes for AliyunOss integration. Contributed by Genmao Yu

This commit is contained in:
Kai Zheng 2016-09-20 11:39:02 +08:00
parent 005f4528c7
commit 9cd4760257
10 changed files with 144 additions and 98 deletions

View File

@ -0,0 +1,87 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.aliyun.oss;
import com.aliyun.oss.common.auth.Credentials;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.aliyun.oss.common.auth.DefaultCredentials;
import com.aliyun.oss.common.auth.InvalidCredentialsException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import java.io.IOException;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
/**
* Support session credentials for authenticating with Aliyun.
*/
public class AliyunCredentialsProvider implements CredentialsProvider {
private Credentials credentials = null;
public AliyunCredentialsProvider(Configuration conf)
throws IOException {
String accessKeyId;
String accessKeySecret;
String securityToken;
try {
accessKeyId = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_ID);
accessKeySecret = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_SECRET);
} catch (IOException e) {
throw new InvalidCredentialsException(e);
}
try {
securityToken = AliyunOSSUtils.getValueWithKey(conf, SECURITY_TOKEN);
} catch (IOException e) {
securityToken = null;
}
if (StringUtils.isEmpty(accessKeyId)
|| StringUtils.isEmpty(accessKeySecret)) {
throw new InvalidCredentialsException(
"AccessKeyId and AccessKeySecret should not be null or empty.");
}
if (StringUtils.isNotEmpty(securityToken)) {
credentials = new DefaultCredentials(accessKeyId, accessKeySecret,
securityToken);
} else {
credentials = new DefaultCredentials(accessKeyId, accessKeySecret);
}
}
@Override
public void setCredentials(Credentials creds) {
if (creds == null) {
throw new InvalidCredentialsException("Credentials should not be null.");
}
credentials = creds;
}
@Override
public Credentials getCredentials() {
if (credentials == null) {
throw new InvalidCredentialsException("Invalid credentials");
}
return credentials;
}
}

View File

@ -18,8 +18,6 @@
package org.apache.hadoop.fs.aliyun.oss;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
@ -43,6 +41,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
/**
* Implementation of {@link FileSystem} for <a href="https://oss.aliyun.com">
* Aliyun OSS</a>, used to access OSS blob system in a filesystem style.

View File

@ -23,7 +23,25 @@
import com.aliyun.oss.OSSException;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.aliyun.oss.common.comm.Protocol;
import com.aliyun.oss.model.*;
import com.aliyun.oss.model.AbortMultipartUploadRequest;
import com.aliyun.oss.model.CannedAccessControlList;
import com.aliyun.oss.model.CompleteMultipartUploadRequest;
import com.aliyun.oss.model.CompleteMultipartUploadResult;
import com.aliyun.oss.model.CopyObjectResult;
import com.aliyun.oss.model.DeleteObjectsRequest;
import com.aliyun.oss.model.GetObjectRequest;
import com.aliyun.oss.model.InitiateMultipartUploadRequest;
import com.aliyun.oss.model.InitiateMultipartUploadResult;
import com.aliyun.oss.model.ListObjectsRequest;
import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.ObjectListing;
import com.aliyun.oss.model.OSSObjectSummary;
import com.aliyun.oss.model.PartETag;
import com.aliyun.oss.model.PutObjectResult;
import com.aliyun.oss.model.UploadPartCopyRequest;
import com.aliyun.oss.model.UploadPartCopyResult;
import com.aliyun.oss.model.UploadPartRequest;
import com.aliyun.oss.model.UploadPartResult;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;

View File

@ -18,8 +18,6 @@
package org.apache.hadoop.fs.aliyun.oss;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
@ -31,6 +29,8 @@
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileSystem.Statistics;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
/**
* The input stream for OSS blob system.
* The class uses multi-part downloading to read data from the object content

View File

@ -18,8 +18,6 @@
package org.apache.hadoop.fs.aliyun.oss;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
@ -33,6 +31,8 @@
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.util.Progressable;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
/**
* The output stream for OSS blob system.
* Data will be buffered on local disk, then uploaded to OSS in

View File

@ -23,8 +23,6 @@
import java.net.URI;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.aliyun.oss.common.auth.DefaultCredentialProvider;
import com.aliyun.oss.common.auth.DefaultCredentials;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.ProviderUtils;
@ -32,7 +30,6 @@
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY;
/**
* Utility methods for Aliyun OSS code.
@ -52,7 +49,7 @@ private AliyunOSSUtils() {
* @return the value for the key
* @throws IOException if failed to get password from configuration
*/
static public String getPassword(Configuration conf, String key)
public static String getValueWithKey(Configuration conf, String key)
throws IOException {
try {
final char[] pass = conf.getPassword(key);
@ -126,12 +123,7 @@ public static CredentialsProvider getCredentialsProvider(URI name,
Configuration newConf =
ProviderUtils.excludeIncompatibleCredentialProviders(conf,
AliyunOSSFileSystem.class);
String accessKey =
AliyunOSSUtils.getPassword(newConf, ACCESS_KEY);
String secretKey =
AliyunOSSUtils.getPassword(newConf, SECRET_KEY);
credentials = new DefaultCredentialProvider(
new DefaultCredentials(accessKey, secretKey));
credentials = new AliyunCredentialsProvider(newConf);
} else {
try {
LOG.debug("Credential provider class is:" + className);
@ -139,7 +131,7 @@ public static CredentialsProvider getCredentialsProvider(URI name,
try {
credentials =
(CredentialsProvider)credClass.getDeclaredConstructor(
URI.class, Configuration.class).newInstance(uri, conf);
Configuration.class).newInstance(conf);
} catch (NoSuchMethodException | SecurityException e) {
credentials =
(CredentialsProvider)credClass.getDeclaredConstructor()

View File

@ -31,8 +31,8 @@ private Constants() {
"fs.oss.credentials.provider";
// OSS access verification
public static final String ACCESS_KEY = "fs.oss.accessKeyId";
public static final String SECRET_KEY = "fs.oss.accessKeySecret";
public static final String ACCESS_KEY_ID = "fs.oss.accessKeyId";
public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret";
public static final String SECURITY_TOKEN = "fs.oss.securityToken";
// Number of simultaneous connections to oss

View File

@ -1,64 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.aliyun.oss;
import com.aliyun.oss.common.auth.Credentials;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.aliyun.oss.common.auth.DefaultCredentials;
import com.aliyun.oss.common.auth.InvalidCredentialsException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import java.net.URI;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
/**
* Support session credentials for authenticating with ALiyun.
*/
public class TemporaryAliyunCredentialsProvider implements CredentialsProvider {
public static final String NAME
= "org.apache.hadoop.fs.aliyun.oss.TemporaryAliyunCredentialsProvider";
private final String accessKeyId;
private final String accessKeySecret;
private final String securityToken;
public TemporaryAliyunCredentialsProvider(URI uri, Configuration conf) {
this.accessKeyId = conf.get(ACCESS_KEY, null);
this.accessKeySecret = conf.get(SECRET_KEY, null);
this.securityToken = conf.get(SECURITY_TOKEN, null);
}
@Override
public void setCredentials(Credentials creds) {
}
@Override
public Credentials getCredentials() {
if (!StringUtils.isEmpty(accessKeyId)
&& !StringUtils.isEmpty(accessKeySecret)
&& !StringUtils.isEmpty(securityToken)) {
return new DefaultCredentials(accessKeyId, accessKeySecret,
securityToken);
}
throw new InvalidCredentialsException(
"AccessKeyId, AccessKeySecret or SecurityToken is unset");
}
}

View File

@ -26,10 +26,10 @@
import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
import org.junit.Test;
import java.net.URI;
import java.io.IOException;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY;
import static org.apache.hadoop.fs.aliyun.oss.Constants.SECRET_KEY;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET;
import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
/**
@ -38,8 +38,7 @@
* should only be used against transient filesystems where you don't care about
* the data.
*/
public class TestAliyunOSSTemporaryCredentials
extends AbstractFSContractTestBase {
public class TestAliyunCredentials extends AbstractFSContractTestBase {
@Override
protected AbstractFSContract createContract(Configuration conf) {
@ -47,19 +46,33 @@ protected AbstractFSContract createContract(Configuration conf) {
}
@Test
public void testTemporaryCredentialValidation() throws Throwable {
public void testCredentialMissingAccessKeyId() throws Throwable {
Configuration conf = new Configuration();
conf.set(ACCESS_KEY, "accessKeyId");
conf.set(SECRET_KEY, "accessKeySecret");
conf.set(SECURITY_TOKEN, "");
URI uri = getFileSystem().getUri();
TemporaryAliyunCredentialsProvider provider
= new TemporaryAliyunCredentialsProvider(uri, conf);
conf.set(ACCESS_KEY_ID, "");
conf.set(ACCESS_KEY_SECRET, "accessKeySecret");
conf.set(SECURITY_TOKEN, "token");
validateCredential(conf);
}
@Test
public void testCredentialMissingAccessKeySecret() throws Throwable {
Configuration conf = new Configuration();
conf.set(ACCESS_KEY_ID, "accessKeyId");
conf.set(ACCESS_KEY_SECRET, "");
conf.set(SECURITY_TOKEN, "token");
validateCredential(conf);
}
private void validateCredential(Configuration conf) {
try {
AliyunCredentialsProvider provider
= new AliyunCredentialsProvider(conf);
Credentials credentials = provider.getCredentials();
fail("Expected a CredentialInitializationException, got " + credentials);
} catch (InvalidCredentialsException expected) {
// expected
} catch (IOException e) {
fail("Unexpected exception.");
}
}
}

View File

@ -67,8 +67,8 @@ public void tearDown() throws Exception {
@BeforeClass
public static void checkSettings() throws Exception {
Configuration conf = new Configuration();
assumeNotNull(conf.get(Constants.ACCESS_KEY));
assumeNotNull(conf.get(Constants.SECRET_KEY));
assumeNotNull(conf.get(Constants.ACCESS_KEY_ID));
assumeNotNull(conf.get(Constants.ACCESS_KEY_SECRET));
assumeNotNull(conf.get("test.fs.oss.name"));
}