big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu."

Browse Source 
This reverts commit e4fca6aae4.

Revert "HADOOP-15671. AliyunOSS: Support Assume Roles in AliyunOSS. Contributed by Jinhu Wu."

This reverts commit 2b635125fb.
This commit is contained in:
Sunil G 2019-01-04 20:35:17 +05:30 committed by Akira Ajisaka
parent 999da98d67
commit 1f425271a7
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
8 changed files with 15 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-project/pom.xml
View File

@ -1279,7 +1279,7 @@
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>3.0.0</version>
<version>2.8.3</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>

5
hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java
View File

@ -120,8 +120,7 @@ public synchronized void close() throws IOException {
if (null == partETags) {
throw new IOException("Failed to multipart upload to oss, abort it.");
}
store.completeMultipartUpload(key, uploadId,
new ArrayList<>(partETags));
store.completeMultipartUpload(key, uploadId, partETags);
}
} finally {
removePartFiles();
@ -130,7 +129,7 @@ public synchronized void close() throws IOException {
}
@Override
public synchronized void write(int b) throws IOException {
public void write(int b) throws IOException {
singleByte[0] = (byte)b;
write(singleByte, 0, 1);
}

5
hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
View File

@ -150,7 +150,7 @@ public void initialize(URI uri, Configuration conf, String user,
"null or empty. Please set proper endpoint with 'fs.oss.endpoint'.");
}
CredentialsProvider provider =
AliyunOSSUtils.getCredentialsProvider(uri, conf);
AliyunOSSUtils.getCredentialsProvider(conf);
ossClient = new OSSClient(endPoint, provider, clientConf);
uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(conf,
MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT);
@ -158,8 +158,6 @@ public void initialize(URI uri, Configuration conf, String user,
serverSideEncryptionAlgorithm =
conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, "");
bucketName = uri.getHost();
String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT);
if (StringUtils.isNotEmpty(cannedACLName)) {
CannedAccessControlList cannedACL =
@ -169,6 +167,7 @@ public void initialize(URI uri, Configuration conf, String user,
}
maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
bucketName = uri.getHost();
}
/**

8
hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
View File

@ -20,7 +20,6 @@
import java.io.File;
import java.io.IOException;
import java.net.URI;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.google.common.base.Preconditions;
@ -96,14 +95,13 @@ public static long calculatePartSize(long contentLength, long minPartSize) {
* Create credential provider specified by configuration, or create default
* credential provider if not specified.
*
* @param uri uri passed by caller
* @param conf configuration
* @return a credential provider
* @throws IOException on any problem. Class construction issues may be
* nested inside the IOE.
*/
public static CredentialsProvider getCredentialsProvider(
URI uri, Configuration conf) throws IOException {
public static CredentialsProvider getCredentialsProvider(Configuration conf)
throws IOException {
CredentialsProvider credentials;
String className = conf.getTrimmed(CREDENTIALS_PROVIDER_KEY);
@ -119,7 +117,7 @@ public static CredentialsProvider getCredentialsProvider(
try {
credentials =
(CredentialsProvider)credClass.getDeclaredConstructor(
URI.class, Configuration.class).newInstance(uri, conf);
Configuration.class).newInstance(conf);
} catch (NoSuchMethodException | SecurityException e) {
credentials =
(CredentialsProvider)credClass.getDeclaredConstructor()

115
hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AssumedRoleCredentialProvider.java
View File

@ -1,115 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.aliyun.oss;
import com.aliyun.oss.common.auth.Credentials;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.aliyun.oss.common.auth.InvalidCredentialsException;
import com.aliyun.oss.common.auth.STSAssumeRoleSessionCredentialsProvider;
import com.aliyuncs.exceptions.ClientException;
import com.aliyuncs.profile.DefaultProfile;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET;
/**
* Support assumed role credentials for authenticating with Aliyun.
* roleArn is configured in core-site.xml
*/
public class AssumedRoleCredentialProvider implements CredentialsProvider {
private static final Logger LOG =
LoggerFactory.getLogger(AssumedRoleCredentialProvider.class);
public static final String NAME
= "org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider";
private Credentials credentials;
private String roleArn;
private long duration;
private String stsEndpoint;
private String sessionName;
private double expiredFactor;
private STSAssumeRoleSessionCredentialsProvider stsCredentialsProvider;
public AssumedRoleCredentialProvider(URI uri, Configuration conf) {
roleArn = conf.getTrimmed(Constants.ROLE_ARN, "");
if (StringUtils.isEmpty(roleArn)) {
throw new InvalidCredentialsException(
"fs.oss.assumed.role.arn is empty");
}
duration = conf.getLong(Constants.ASSUMED_ROLE_DURATION,
Constants.ASSUMED_ROLE_DURATION_DEFAULT);
expiredFactor = conf.getDouble(Constants.ASSUMED_ROLE_STS_EXPIRED_FACTOR,
Constants.ASSUMED_ROLE_STS_EXPIRED_FACTOR_DEFAULT);
stsEndpoint = conf.getTrimmed(Constants.ASSUMED_ROLE_STS_ENDPOINT, "");
if (StringUtils.isEmpty(stsEndpoint)) {
throw new InvalidCredentialsException(
"fs.oss.assumed.role.sts.endpoint is empty");
}
sessionName = conf.getTrimmed(Constants.ASSUMED_ROLE_SESSION_NAME, "");
String accessKeyId;
String accessKeySecret;
try {
accessKeyId = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_ID);
accessKeySecret = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_SECRET);
} catch (IOException e) {
throw new InvalidCredentialsException(e);
}
try {
DefaultProfile.addEndpoint("", "", "Sts", stsEndpoint);
} catch (ClientException e) {
throw new InvalidCredentialsException(e);
}
stsCredentialsProvider = new STSAssumeRoleSessionCredentialsProvider(
new com.aliyuncs.auth.BasicCredentials(accessKeyId, accessKeySecret),
roleArn, DefaultProfile.getProfile("", accessKeyId, accessKeySecret))
.withExpiredDuration(duration).withExpiredFactor(expiredFactor);
if (!StringUtils.isEmpty(sessionName)) {
stsCredentialsProvider.withRoleSessionName(sessionName);
}
}
@Override
public void setCredentials(Credentials creds) {
throw new InvalidCredentialsException(
"Should not set credentials from external call");
}
@Override
public Credentials getCredentials() {
credentials = stsCredentialsProvider.getCredentials();
if (credentials == null) {
throw new InvalidCredentialsException("Invalid credentials");
}
return credentials;
}
}

22
hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.fs.aliyun.oss;
import com.aliyun.oss.common.utils.AuthUtils;
import com.aliyun.oss.common.utils.VersionInfoUtils;
/**
@ -45,27 +44,6 @@ private Constants() {
public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret";
public static final String SECURITY_TOKEN = "fs.oss.securityToken";
// Assume role configurations
public static final String ROLE_ARN = "fs.oss.assumed.role.arn";
public static final String ASSUMED_ROLE_DURATION =
"fs.oss.assumed.role.session.duration";
// Default session duration(in seconds)
public static final long ASSUMED_ROLE_DURATION_DEFAULT = 30 * 60;
// Expired factor of sts token
// For example, if session duration is 900s and expiredFactor is 0.8
// sts token will be refreshed after 900 * 0.8s
public static final String ASSUMED_ROLE_STS_EXPIRED_FACTOR =
"fs.oss.assumed.role.sts.expiredFactor";
public static final double ASSUMED_ROLE_STS_EXPIRED_FACTOR_DEFAULT =
AuthUtils.DEFAULT_EXPIRED_FACTOR;
public static final String ASSUMED_ROLE_STS_ENDPOINT =
"fs.oss.assumed.role.sts.endpoint";
public static final String ASSUMED_ROLE_SESSION_NAME =
"fs.oss.assumed.role.session.name";
// Number of simultaneous connections to oss
public static final String MAXIMUM_CONNECTIONS_KEY =
"fs.oss.connection.maximum";

54
hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
View File

@ -128,58 +128,8 @@ please raise your issues with them.
</property>
<property>
<name>fs.oss.impl</name>
<value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
</property>
<property>
<name>fs.oss.assumed.role.arn</name>
<description>
Role ARN for the role to be assumed.
Required if the fs.oss.credentials.provider is
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
</description>
</property>
<property>
<name>fs.oss.assumed.role.sts.endpoint</name>
<description>
STS Token Service endpoint.
Required if the fs.oss.credentials.provider is
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
</description>
</property>
<property>
<name>fs.oss.assumed.role.session.name</name>
<value />
<description>
Session name for the assumed role, must be valid characters
according to Aliyun API. It is optional, will be generated by
oss java sdk if it is empty.
Only used if the fs.oss.credentials.provider is
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
</description>
</property>
<property>
<name>fs.oss.assumed.role.session.duration</name>
<value />
<description>
Duration of assumed roles before it is expired. Default is 30 minutes.
Only used if the fs.oss.credentials.provider is
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
</description>
</property>
<property>
<name>fs.oss.assumed.role.sts.expiredFactor</name>
<value />
<description>
Sts token will be refreshed after (expiredFactor * duration) seconds.
Only used if the fs.oss.credentials.provider is
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
</description>
<name>fs.oss.impl</name>
<value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
</property>
<property>

53
hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java
View File

@ -19,7 +19,6 @@
package org.apache.hadoop.fs.aliyun.oss;
import com.aliyun.oss.common.auth.Credentials;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.aliyun.oss.common.auth.InvalidCredentialsException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract;
@ -28,15 +27,9 @@
import org.junit.Test;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_SESSION_NAME;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_STS_ENDPOINT;
import static org.apache.hadoop.fs.aliyun.oss.Constants.CREDENTIALS_PROVIDER_KEY;
import static org.apache.hadoop.fs.aliyun.oss.Constants.ROLE_ARN;
import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
/**
@ -70,54 +63,16 @@ public void testCredentialMissingAccessKeySecret() throws Throwable {
validateCredential(conf);
}
@Test
public void testCredentialMissingRoleArn() throws Throwable {
Configuration conf = new Configuration();
conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
conf.set(ROLE_ARN, "");
validateCredential(conf);
}
@Test
public void testCredentialMissingStsEndpoint() throws Throwable {
Configuration conf = new Configuration();
conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
conf.set(ASSUMED_ROLE_STS_ENDPOINT, "");
validateCredential(conf);
}
@Test
public void testCredentialInvalidSessionName() throws Throwable {
Configuration conf = new Configuration();
conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
conf.set(ASSUMED_ROLE_SESSION_NAME, "hadoop oss");
validateCredential(conf);
}
private void validateCredential(URI uri, Configuration conf) {
private void validateCredential(Configuration conf) {
try {
CredentialsProvider provider =
AliyunOSSUtils.getCredentialsProvider(uri, conf);
AliyunCredentialsProvider provider
= new AliyunCredentialsProvider(conf);
Credentials credentials = provider.getCredentials();
fail("Expected a CredentialInitializationException, got " + credentials);
} catch (InvalidCredentialsException expected) {
// expected
} catch (IOException e) {
Throwable cause = e.getCause();
if (cause instanceof InvocationTargetException) {
boolean isInstance =
((InvocationTargetException)cause).getTargetException()
instanceof InvalidCredentialsException;
if (!isInstance) {
fail("Unexpected exception.");
}
} else {
fail("Unexpected exception.");
}
fail("Unexpected exception.");
}
}
private void validateCredential(Configuration conf) {
validateCredential(null, conf);
}
}