HDDS-939. Add S3 access check to Ozone manager. Contributed by Ajay Kumar. (#634)
This commit is contained in:
parent
5c0a81ad3c
commit
82d477293c
@ -61,8 +61,7 @@ public S3SecretManagerImpl(OzoneConfiguration configuration,
|
||||
public S3SecretValue getS3Secret(String kerberosID) throws IOException {
|
||||
Preconditions.checkArgument(Strings.isNotBlank(kerberosID),
|
||||
"kerberosID cannot be null or empty.");
|
||||
String awsAccessKeyStr = DigestUtils.md5Hex(kerberosID);
|
||||
byte[] awsAccessKey = awsAccessKeyStr.getBytes(UTF_8);
|
||||
byte[] awsAccessKey = kerberosID.getBytes(UTF_8);
|
||||
S3SecretValue result = null;
|
||||
omMetadataManager.getLock().acquireS3SecretLock(kerberosID);
|
||||
try {
|
||||
@ -77,33 +76,31 @@ public S3SecretValue getS3Secret(String kerberosID) throws IOException {
|
||||
result = S3SecretValue.fromProtobuf(
|
||||
OzoneManagerProtocolProtos.S3Secret.parseFrom(s3Secret));
|
||||
}
|
||||
result.setAwsAccessKey(awsAccessKeyStr);
|
||||
} finally {
|
||||
omMetadataManager.getLock().releaseS3SecretLock(kerberosID);
|
||||
}
|
||||
LOG.trace("Secret for kerberosID:{},accessKey:{}, proto:{}", kerberosID,
|
||||
awsAccessKeyStr, result);
|
||||
LOG.trace("Secret for accessKey:{}, proto:{}", kerberosID, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getS3UserSecretString(String awsAccessKeyId)
|
||||
public String getS3UserSecretString(String kerberosID)
|
||||
throws IOException {
|
||||
Preconditions.checkArgument(Strings.isNotBlank(awsAccessKeyId),
|
||||
Preconditions.checkArgument(Strings.isNotBlank(kerberosID),
|
||||
"awsAccessKeyId cannot be null or empty.");
|
||||
LOG.trace("Get secret for awsAccessKey:{}", awsAccessKeyId);
|
||||
LOG.trace("Get secret for awsAccessKey:{}", kerberosID);
|
||||
|
||||
byte[] s3Secret;
|
||||
omMetadataManager.getLock().acquireS3SecretLock(awsAccessKeyId);
|
||||
omMetadataManager.getLock().acquireS3SecretLock(kerberosID);
|
||||
try {
|
||||
s3Secret = omMetadataManager.getS3SecretTable()
|
||||
.get(awsAccessKeyId.getBytes(UTF_8));
|
||||
.get(kerberosID.getBytes(UTF_8));
|
||||
if (s3Secret == null) {
|
||||
throw new OzoneSecurityException("S3 secret not found for " +
|
||||
"awsAccessKeyId " + awsAccessKeyId, S3_SECRET_NOT_FOUND);
|
||||
"awsAccessKeyId " + kerberosID, S3_SECRET_NOT_FOUND);
|
||||
}
|
||||
} finally {
|
||||
omMetadataManager.getLock().releaseS3SecretLock(awsAccessKeyId);
|
||||
omMetadataManager.getLock().releaseS3SecretLock(kerberosID);
|
||||
}
|
||||
|
||||
return OzoneManagerProtocolProtos.S3Secret.parseFrom(s3Secret)
|
||||
|
@ -17,7 +17,6 @@
|
||||
*/
|
||||
package org.apache.hadoop.ozone.om.helpers;
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
|
||||
|
||||
/**
|
||||
@ -26,12 +25,10 @@
|
||||
public class S3SecretValue {
|
||||
private String kerberosID;
|
||||
private String awsSecret;
|
||||
private String awsAccessKey;
|
||||
|
||||
public S3SecretValue(String kerberosID, String awsSecret) {
|
||||
this.kerberosID = kerberosID;
|
||||
this.awsSecret = awsSecret;
|
||||
this.awsAccessKey = DigestUtils.md5Hex(kerberosID);
|
||||
}
|
||||
|
||||
public String getKerberosID() {
|
||||
@ -51,11 +48,7 @@ public void setAwsSecret(String awsSecret) {
|
||||
}
|
||||
|
||||
public String getAwsAccessKey() {
|
||||
return awsAccessKey;
|
||||
}
|
||||
|
||||
public void setAwsAccessKey(String awsAccessKey) {
|
||||
this.awsAccessKey = awsAccessKey;
|
||||
return kerberosID;
|
||||
}
|
||||
|
||||
public static S3SecretValue fromProtobuf(
|
||||
@ -72,6 +65,6 @@ public OzoneManagerProtocolProtos.S3Secret getProtobuf() {
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "awsAccessKey=" + awsAccessKey + "\nawsSecret=" + awsSecret;
|
||||
return "awsAccessKey=" + kerberosID + "\nawsSecret=" + awsSecret;
|
||||
}
|
||||
}
|
||||
|
@ -81,6 +81,7 @@
|
||||
import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
|
||||
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
|
||||
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
|
||||
import org.apache.hadoop.ozone.s3.util.OzoneS3Util;
|
||||
import org.apache.hadoop.test.GenericTestUtils;
|
||||
import org.apache.hadoop.test.LambdaTestUtils;
|
||||
import org.apache.hadoop.util.Time;
|
||||
@ -92,6 +93,8 @@
|
||||
import static org.hamcrest.CoreMatchers.containsString;
|
||||
import static org.hamcrest.CoreMatchers.either;
|
||||
import org.junit.Assert;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertThat;
|
||||
@ -288,6 +291,23 @@ public void testCreateS3Bucket()
|
||||
Assert.assertTrue(volume.getCreationTime() >= currentTime);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateSecureS3Bucket() throws IOException {
|
||||
long currentTime = Time.now();
|
||||
String userName = "ozone/localhost@EXAMPLE.COM";
|
||||
String bucketName = UUID.randomUUID().toString();
|
||||
String s3VolumeName = OzoneS3Util.getVolumeName(userName);
|
||||
store.createS3Bucket(s3VolumeName, bucketName);
|
||||
String volumeName = store.getOzoneVolumeName(bucketName);
|
||||
assertEquals(volumeName, "s3" + s3VolumeName);
|
||||
|
||||
OzoneVolume volume = store.getVolume(volumeName);
|
||||
OzoneBucket bucket = volume.getBucket(bucketName);
|
||||
Assert.assertEquals(bucketName, bucket.getName());
|
||||
Assert.assertTrue(bucket.getCreationTime() >= currentTime);
|
||||
Assert.assertTrue(volume.getCreationTime() >= currentTime);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testListS3Buckets()
|
||||
|
@ -53,6 +53,8 @@
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import static org.apache.hadoop.ozone.s3.util.OzoneS3Util.getVolumeName;
|
||||
import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.slf4j.Logger;
|
||||
@ -196,9 +198,10 @@ public Response list(
|
||||
public Response put(@PathParam("bucket") String bucketName, @Context
|
||||
HttpHeaders httpHeaders) throws IOException, OS3Exception {
|
||||
|
||||
String userName = getAuthenticationHeaderParser().getAccessKeyID();
|
||||
String volumeName = getVolumeName(getAuthenticationHeaderParser().
|
||||
getAccessKeyID());
|
||||
|
||||
String location = createS3Bucket(userName, bucketName);
|
||||
String location = createS3Bucket(volumeName, bucketName);
|
||||
|
||||
LOG.info("Location is {}", location);
|
||||
return Response.status(HttpStatus.SC_OK).header("Location", location)
|
||||
|
@ -34,6 +34,8 @@
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.apache.hadoop.ozone.s3.util.OzoneS3Util.getVolumeName;
|
||||
|
||||
/**
|
||||
* Top level rest endpoint.
|
||||
*/
|
||||
@ -63,8 +65,9 @@ public Response get()
|
||||
.header("Location", "/static/")
|
||||
.build();
|
||||
}
|
||||
String userName = authenticationHeaderParser.getAccessKeyID();
|
||||
Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(userName,
|
||||
String volumeName = getVolumeName(authenticationHeaderParser.
|
||||
getAccessKeyID());
|
||||
Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(volumeName,
|
||||
null);
|
||||
|
||||
while (bucketIterator.hasNext()) {
|
||||
|
@ -59,13 +59,25 @@ public class Credential {
|
||||
@SuppressWarnings("StringSplitter")
|
||||
public void parseCredential() throws OS3Exception {
|
||||
String[] split = credential.split("/");
|
||||
if (split.length == 5) {
|
||||
switch (split.length) {
|
||||
case 5:
|
||||
// Ex: dkjad922329ddnks/20190321/us-west-1/s3/aws4_request
|
||||
accessKeyID = split[0].trim();
|
||||
date = split[1].trim();
|
||||
awsRegion = split[2].trim();
|
||||
awsService = split[3].trim();
|
||||
awsRequest = split[4].trim();
|
||||
} else {
|
||||
return;
|
||||
case 6:
|
||||
// Access id is kerberos principal.
|
||||
// Ex: testuser/om@EXAMPLE.COM/20190321/us-west-1/s3/aws4_request
|
||||
accessKeyID = split[0] + "/" +split[1];
|
||||
date = split[2].trim();
|
||||
awsRegion = split[3].trim();
|
||||
awsService = split[4].trim();
|
||||
awsRequest = split[5].trim();
|
||||
return;
|
||||
default:
|
||||
LOG.error("Credentials not in expected format. credential:{}",
|
||||
credential);
|
||||
throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_HEADER, credential);
|
||||
|
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
package org.apache.hadoop.ozone.s3.util;
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Ozone util for S3 related operations.
|
||||
*/
|
||||
public final class OzoneS3Util {
|
||||
|
||||
private OzoneS3Util() {
|
||||
}
|
||||
|
||||
public static String getVolumeName(String userName) {
|
||||
Objects.requireNonNull(userName);
|
||||
return DigestUtils.md5Hex(userName);
|
||||
}
|
||||
}
|
@ -25,6 +25,7 @@
|
||||
import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import org.apache.hadoop.ozone.s3.util.OzoneS3Util;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
@ -61,10 +62,11 @@ public void testListBucket() throws Exception {
|
||||
ListBucketResponse response =
|
||||
(ListBucketResponse) rootEndpoint.get().getEntity();
|
||||
assertEquals(0, response.getBucketsNum());
|
||||
String volumeName = OzoneS3Util.getVolumeName(userName);
|
||||
|
||||
String bucketBaseName = "bucket-" + getClass().getName();
|
||||
for(int i = 0; i < 10; i++) {
|
||||
objectStoreStub.createS3Bucket(userName, bucketBaseName + i);
|
||||
objectStoreStub.createS3Bucket(volumeName, bucketBaseName + i);
|
||||
}
|
||||
response = (ListBucketResponse) rootEndpoint.get().getEntity();
|
||||
assertEquals(10, response.getBucketsNum());
|
||||
|
Loading…
Reference in New Issue
Block a user