From 4028cac56d469c566f2dbad9e9f11c36c53f5ee9 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Thu, 22 Aug 2019 15:00:17 -0700 Subject: [PATCH] HDDS-1347. In OM HA getS3Secret call Should happen only leader OM. (#670) --- .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../apache/hadoop/ozone/audit/OMAction.java | 4 +- .../ozone/om/exceptions/OMException.java | 5 +- .../src/main/proto/OzoneManagerProtocol.proto | 14 ++ .../hadoop/ozone/TestSecureOzoneCluster.java | 12 +- .../apache/hadoop/ozone/om/OzoneManager.java | 8 + .../s3/security/S3GetSecretRequest.java | 193 ++++++++++++++++++ .../om/request/s3/security/package-info.java | 22 ++ .../s3/security/S3GetSecretResponse.java | 56 +++++ .../om/response/s3/security/package-info.java | 22 ++ 10 files changed, 333 insertions(+), 4 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3GetSecretRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/package-info.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3GetSecretResponse.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index d9b33d8341..80e9260c71 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -268,6 +268,7 @@ private OzoneConsts() { public static final String PART_NUMBER_MARKER = "partNumberMarker"; public static final String MAX_PARTS = "maxParts"; public static final String S3_BUCKET = "s3Bucket"; + public static final String S3_GETSECRET_USER = "S3GetSecretUser"; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java index ebcd439095..97d4afc46e 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java @@ -69,7 +69,9 @@ public enum OMAction implements AuditAction { CREATE_DIRECTORY, CREATE_FILE, LOOKUP_FILE, - LIST_STATUS; + LIST_STATUS, + + GET_S3_SECRET; @Override public String getAction() { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 1e291edfcd..268471a62c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -207,7 +207,10 @@ public enum ResultCodes { RATIS_ERROR, // Error in Ratis server - INVALID_PATH_IN_ACL_REQUEST // Error code when path name is invalid during + INVALID_PATH_IN_ACL_REQUEST, // Error code when path name is invalid during // acl requests. + + USER_MISMATCH // Error code when requested user name passed is different + // from remote user. } } diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto index ded16070bf..7d5f0987bf 100644 --- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto @@ -156,6 +156,8 @@ message OMRequest { optional GetAclRequest getAclRequest = 78; optional PurgeKeysRequest purgeKeysRequest = 81; + + optional UpdateGetS3SecretRequest updateGetS3SecretRequest = 82; } message OMResponse { @@ -287,6 +289,9 @@ enum Status { RATIS_ERROR = 52; INVALID_PATH_IN_ACL_REQUEST = 53; // Invalid path name in acl request. + + USER_MISMATCH = 54; // Error code when requested user name passed is + // different from remote user. } @@ -1050,6 +1055,15 @@ message GetS3SecretResponse { required S3Secret s3Secret = 2; } +/** + This will be used internally by OM to replicate S3 Secret across quorum of + OM's. +*/ +message UpdateGetS3SecretRequest { + required string kerberosID = 1; + required string awsSecret = 2; +} + /** The OM service that takes care of Ozone namespace. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 853b6a2c30..709c43f43c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -104,6 +104,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.slf4j.event.Level.INFO; /** @@ -689,11 +690,11 @@ public void testGetS3Secret() throws Exception { //Creates a secret since it does not exist S3SecretValue firstAttempt = omClient - .getS3Secret("HADOOP/JOHNDOE"); + .getS3Secret(UserGroupInformation.getCurrentUser().getUserName()); //Fetches the secret from db since it was created in previous step S3SecretValue secondAttempt = omClient - .getS3Secret("HADOOP/JOHNDOE"); + .getS3Secret(UserGroupInformation.getCurrentUser().getUserName()); //secret fetched on both attempts must be same assertTrue(firstAttempt.getAwsSecret() @@ -703,6 +704,13 @@ public void testGetS3Secret() throws Exception { assertTrue(firstAttempt.getAwsAccessKey() .equals(secondAttempt.getAwsAccessKey())); + + try { + omClient.getS3Secret("HADOOP/JOHNDOE"); + fail("testGetS3Secret failed"); + } catch (IOException ex) { + GenericTestUtils.assertExceptionContains("USER_MISMATCH", ex); + } } finally { if(om != null){ om.stop(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index dbd5d39881..bbbd61cada 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2683,6 +2683,14 @@ public void deleteS3Bucket(String s3BucketName) throws IOException { * {@inheritDoc} */ public S3SecretValue getS3Secret(String kerberosID) throws IOException{ + UserGroupInformation user = ProtobufRpcEngine.Server.getRemoteUser(); + + // Check whether user name passed is matching with the current user or not. + if (!user.getUserName().equals(kerberosID)) { + throw new OMException("User mismatch. Requested user name is " + + "mismatched " + kerberosID +", with current user " + + user.getUserName(), OMException.ResultCodes.USER_MISMATCH); + } return s3SecretManager.getS3Secret(kerberosID); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3GetSecretRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3GetSecretRequest.java new file mode 100644 index 0000000000..60f808c55a --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3GetSecretRequest.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.s3.security; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import com.google.common.base.Optional; +import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.audit.OMAction; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.S3SecretValue; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.s3.security.S3GetSecretResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UpdateGetS3SecretRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Secret; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.utils.db.cache.CacheKey; +import org.apache.hadoop.utils.db.cache.CacheValue; + +import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.S3_SECRET_LOCK; + +/** + * Handles GetS3Secret request. + */ +public class S3GetSecretRequest extends OMClientRequest { + + private static final Logger LOG = + LoggerFactory.getLogger(S3GetSecretRequest.class); + + public S3GetSecretRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { + GetS3SecretRequest s3GetSecretRequest = + getOmRequest().getGetS3SecretRequest(); + + // Generate S3 Secret to be used by OM quorum. + String kerberosID = s3GetSecretRequest.getKerberosID(); + + UserGroupInformation user = ProtobufRpcEngine.Server.getRemoteUser(); + if (!user.getUserName().equals(kerberosID)) { + throw new OMException("User mismatch. Requested user name is " + + "mismatched " + kerberosID +", with current user " + + user.getUserName(), OMException.ResultCodes.USER_MISMATCH); + } + + String s3Secret = DigestUtils.sha256Hex(OmUtils.getSHADigest()); + + UpdateGetS3SecretRequest updateGetS3SecretRequest = + UpdateGetS3SecretRequest.newBuilder() + .setAwsSecret(s3Secret) + .setKerberosID(kerberosID).build(); + + // Client issues GetS3Secret request, when received by OM leader + // it will generate s3Secret. Original GetS3Secret request is + // converted to UpdateGetS3Secret request with the generated token + // information. This updated request will be submitted to Ratis. In this + // way S3Secret created by leader, will be replicated across all + // OMs. With this approach, original GetS3Secret request from + // client does not need any proto changes. + OMRequest.Builder omRequest = OMRequest.newBuilder() + .setUserInfo(getUserInfo()) + .setUpdateGetS3SecretRequest(updateGetS3SecretRequest) + .setCmdType(getOmRequest().getCmdType()) + .setClientId(getOmRequest().getClientId()); + + if (getOmRequest().hasTraceID()) { + omRequest.setTraceID(getOmRequest().getTraceID()); + } + + return omRequest.build(); + + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long transactionLogIndex, + OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper) { + + + OMClientResponse omClientResponse = null; + OMResponse.Builder omResponse = OMResponse.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.GetS3Secret) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setSuccess(true); + boolean acquiredLock = false; + IOException exception = null; + OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + UpdateGetS3SecretRequest updateGetS3SecretRequest = + getOmRequest().getUpdateGetS3SecretRequest(); + String kerberosID = updateGetS3SecretRequest.getKerberosID(); + try { + String awsSecret = updateGetS3SecretRequest.getAwsSecret(); + acquiredLock = + omMetadataManager.getLock().acquireLock(S3_SECRET_LOCK, kerberosID); + + S3SecretValue s3SecretValue = + omMetadataManager.getS3SecretTable().get(kerberosID); + + // If s3Secret for user is not in S3Secret table, add the Secret to cache. + if (s3SecretValue == null) { + omMetadataManager.getS3SecretTable().addCacheEntry( + new CacheKey<>(kerberosID), + new CacheValue<>(Optional.of(new S3SecretValue(kerberosID, + awsSecret)), transactionLogIndex)); + } else { + // If it already exists, use the existing one. + awsSecret = s3SecretValue.getAwsSecret(); + } + + GetS3SecretResponse.Builder getS3SecretResponse = GetS3SecretResponse + .newBuilder().setS3Secret(S3Secret.newBuilder() + .setAwsSecret(awsSecret).setKerberosID(kerberosID)); + + if (s3SecretValue == null) { + omClientResponse = + new S3GetSecretResponse(new S3SecretValue(kerberosID, awsSecret), + omResponse.setGetS3SecretResponse(getS3SecretResponse).build()); + } else { + // As when it already exists, we don't need to add to DB again. So + // set the value to null. + omClientResponse = new S3GetSecretResponse(null, + omResponse.setGetS3SecretResponse(getS3SecretResponse).build()); + } + + } catch (IOException ex) { + exception = ex; + omClientResponse = new S3GetSecretResponse(null, + createErrorOMResponse(omResponse, ex)); + } finally { + if (omClientResponse != null) { + omClientResponse.setFlushFuture(ozoneManagerDoubleBufferHelper.add( + omClientResponse, transactionLogIndex)); + } + if (acquiredLock) { + omMetadataManager.getLock().releaseLock(S3_SECRET_LOCK, kerberosID); + } + } + + + Map auditMap = new HashMap<>(); + auditMap.put(OzoneConsts.S3_GETSECRET_USER, kerberosID); + + // audit log + auditLog(ozoneManager.getAuditLogger(), buildAuditMessage( + OMAction.GET_S3_SECRET, auditMap, + exception, getOmRequest().getUserInfo())); + + if (exception == null) { + LOG.debug("Secret for accessKey:{} is generated Successfully", + kerberosID); + } else { + LOG.error("Secret for accessKey:{} is generation failed", kerberosID, + exception); + } + return omClientResponse; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/package-info.java new file mode 100644 index 0000000000..94a6b11686 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package contains classes related to S3 security requests. + */ +package org.apache.hadoop.ozone.om.request.s3.security; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3GetSecretResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3GetSecretResponse.java new file mode 100644 index 0000000000..61e20160e2 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3GetSecretResponse.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.s3.security; + +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.S3SecretValue; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.utils.db.BatchOperation; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; + +/** + * Response for GetS3Secret request. + */ +public class S3GetSecretResponse extends OMClientResponse { + + + private S3SecretValue s3SecretValue; + + public S3GetSecretResponse(@Nullable S3SecretValue s3SecretValue, + @Nonnull OMResponse omResponse) { + super(omResponse); + this.s3SecretValue = s3SecretValue; + } + + @Override + public void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + + if (s3SecretValue != null && + getOMResponse().getStatus() == OzoneManagerProtocolProtos.Status.OK) { + omMetadataManager.getS3SecretTable().putWithBatch(batchOperation, + s3SecretValue.getKerberosID(), s3SecretValue); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java new file mode 100644 index 0000000000..d9024d1c85 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package contains classes related to S3 security responses. + */ +package org.apache.hadoop.ozone.om.request.s3.security;