From 219e2867220f805fb69ff9c991113f92c3f4ed2f Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Thu, 30 May 2019 16:44:38 -0700 Subject: [PATCH] HDDS-1608. Support Ozone Prefix ACLs in OM metadata table. Contributed by Xiaoyu Yao. (#875) --- .../hadoop/ozone/om/OMMetadataManager.java | 7 + .../ozone/om/codec/OmPrefixInfoCodec.java | 53 ++++++ .../hadoop/ozone/om/helpers/OmPrefixInfo.java | 170 ++++++++++++++++++ .../src/main/proto/OzoneManagerProtocol.proto | 7 + .../ozone/om/codec/TestOmPrefixInfoCodec.java | 98 ++++++++++ .../server/ratis/TestCSMMetrics.java | 14 +- .../ozone/om/OmMetadataManagerImpl.java | 19 +- 7 files changed, 360 insertions(+), 8 deletions(-) create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmPrefixInfoCodec.java create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmPrefixInfoCodec.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 3149b86fc0..34d81cf50c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList; @@ -262,6 +263,12 @@ List listVolumes(String userName, String prefix, Table getS3Table(); + /** + * Gets the Ozone prefix path to its acl mapping table. + * @return Table. + */ + Table getPrefixTable(); + /** * Returns the DB key name of a multipart upload key in OM metadata store. * diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmPrefixInfoCodec.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmPrefixInfoCodec.java new file mode 100644 index 0000000000..5ddb468bfc --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmPrefixInfoCodec.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.om.codec; + +import com.google.common.base.Preconditions; +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrefixInfo; + +import org.apache.hadoop.utils.db.Codec; + +import java.io.IOException; + +/** + * Codec to encode PrefixAcl as byte array. + */ +public class OmPrefixInfoCodec implements Codec { + + @Override + public byte[] toPersistedFormat(OmPrefixInfo object) throws IOException { + Preconditions + .checkNotNull(object, "Null object can't be converted to byte array."); + return object.getProtobuf().toByteArray(); + } + + @Override + public OmPrefixInfo fromPersistedFormat(byte[] rawData) throws IOException { + Preconditions + .checkNotNull(rawData, + "Null byte array can't converted to real object."); + try { + return OmPrefixInfo.getFromProtobuf(PrefixInfo.parseFrom(rawData)); + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException( + "Can't encode the the raw data from the byte array", e); + } + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java new file mode 100644 index 0000000000..ddb9865e4d --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrefixInfo; +import org.apache.hadoop.ozone.protocolPB.OMPBHelper; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Wrapper class for Ozone prefix path info, currently mainly target for ACL but + * can be extended for other OzFS optimizations in future. + */ +// TODO: support Auditable interface +public final class OmPrefixInfo extends WithMetadata { + + private String name; + private List acls; + + public OmPrefixInfo(String name, List acls, + Map metadata) { + this.name = name; + this.acls = acls; + this.metadata = metadata; + } + + /** + * Returns the ACL's associated with this prefix. + * @return {@literal List} + */ + public List getAcls() { + return acls; + } + + /** + * Returns the name of the prefix path. + * @return name of the prefix path. + */ + public String getName() { + return name; + } + + /** + * Returns new builder class that builds a OmPrefixInfo. + * + * @return Builder + */ + public static OmPrefixInfo.Builder newBuilder() { + return new OmPrefixInfo.Builder(); + } + + /** + * Builder for OmPrefixInfo. + */ + public static class Builder { + private String name; + private List acls; + private Map metadata; + + public Builder() { + //Default values + this.acls = new LinkedList<>(); + this.metadata = new HashMap<>(); + } + + public Builder setAcls(List listOfAcls) { + this.acls = listOfAcls; + return this; + } + + public Builder setName(String n) { + this.name = n; + return this; + } + + public OmPrefixInfo.Builder addMetadata(String key, String value) { + metadata.put(key, value); + return this; + } + + public OmPrefixInfo.Builder addAllMetadata( + Map additionalMetadata) { + if (additionalMetadata != null) { + metadata.putAll(additionalMetadata); + } + return this; + } + + /** + * Constructs the OmPrefixInfo. + * @return instance of OmPrefixInfo. + */ + public OmPrefixInfo build() { + Preconditions.checkNotNull(name); + Preconditions.checkNotNull(acls); + return new OmPrefixInfo(name, acls, metadata); + } + } + + /** + * Creates PrefixInfo protobuf from OmPrefixInfo. + */ + public PrefixInfo getProtobuf() { + PrefixInfo.Builder pib = PrefixInfo.newBuilder().setName(name) + .addAllAcls(acls.stream().map(OMPBHelper::convertOzoneAcl) + .collect(Collectors.toList())) + .addAllMetadata(KeyValueUtil.toProtobuf(metadata)); + return pib.build(); + } + + /** + * Parses PrefixInfo protobuf and creates OmPrefixInfo. + * @param prefixInfo + * @return instance of OmPrefixInfo + */ + public static OmPrefixInfo getFromProtobuf(PrefixInfo prefixInfo) { + OmPrefixInfo.Builder opib = OmPrefixInfo.newBuilder() + .setName(prefixInfo.getName()) + .setAcls(prefixInfo.getAclsList().stream().map( + OMPBHelper::convertOzoneAcl).collect(Collectors.toList())); + if (prefixInfo.getMetadataList() != null) { + opib.addAllMetadata(KeyValueUtil + .getFromProtobuf(prefixInfo.getMetadataList())); + } + return opib.build(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OmPrefixInfo that = (OmPrefixInfo) o; + return name.equals(that.name) && + Objects.equals(acls, that.acls) && + Objects.equals(metadata, that.metadata); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } +} + diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto index 6577de7368..8e8d401a7f 100644 --- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto @@ -459,11 +459,18 @@ message BucketArgs { repeated hadoop.hdds.KeyValue metadata = 7; } +message PrefixInfo { + required string name = 1; + repeated OzoneAclInfo acls = 2; + repeated hadoop.hdds.KeyValue metadata = 3; +} + message OzoneObj { enum ObjectType { VOLUME = 1; BUCKET = 2; KEY = 3; + PREFIX = 4; } enum StoreType { diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmPrefixInfoCodec.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmPrefixInfoCodec.java new file mode 100644 index 0000000000..571a25f203 --- /dev/null +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmPrefixInfoCodec.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.om.codec; + +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; + +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * This class test OmPrefixInfoCodec. + */ +public class TestOmPrefixInfoCodec { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + + private OmPrefixInfoCodec codec; + + @Before + public void setUp() { + codec = new OmPrefixInfoCodec(); + } + + @Test + public void testCodecWithIncorrectValues() throws Exception { + try { + codec.fromPersistedFormat("random".getBytes(StandardCharsets.UTF_8)); + fail("testCodecWithIncorrectValues failed"); + } catch (IllegalArgumentException ex) { + GenericTestUtils.assertExceptionContains("Can't encode the the raw " + + "data from the byte array", ex); + } + } + + @Test + public void testCodecWithNullDataFromTable() throws Exception { + thrown.expect(NullPointerException.class); + codec.fromPersistedFormat(null); + } + + + @Test + public void testCodecWithNullDataFromUser() throws Exception { + thrown.expect(NullPointerException.class); + codec.toPersistedFormat(null); + } + + @Test + public void testToAndFromPersistedFormat() throws IOException { + + List acls = new LinkedList<>(); + OzoneAcl ozoneAcl = new OzoneAcl(ACLIdentityType.USER, + "hive", ACLType.ALL); + acls.add(ozoneAcl); + OmPrefixInfo opiSave = OmPrefixInfo.newBuilder() + .setName("/user/hive/warehouse") + .setAcls(acls) + .addMetadata("id", "100") + .build(); + + OmPrefixInfo opiLoad = codec.fromPersistedFormat( + codec.toPersistedFormat(opiSave)); + + assertTrue("Load saved prefix info should match", + opiLoad.equals(opiSave)); + } +} \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java index fa740eacc8..5d886c1caa 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java @@ -58,13 +58,13 @@ import org.junit.Test; import org.junit.Assert; - /** - * This class tests the metrics of ContainerStateMachine. - */ - public class TestCSMMetrics { - static final String TEST_DIR = - GenericTestUtils.getTestDir("dfs").getAbsolutePath() - + File.separator; +/** + * This class tests the metrics of ContainerStateMachine. + */ +public class TestCSMMetrics { + static final String TEST_DIR = + GenericTestUtils.getTestDir("dfs").getAbsolutePath() + + File.separator; @FunctionalInterface interface CheckedBiFunction { OUT apply(LEFT left, RIGHT right) throws THROWABLE; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 0720a10b24..fb7c3b8c2c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.om.codec.OmKeyInfoCodec; import org.apache.hadoop.ozone.om.codec.OmMultipartKeyInfoCodec; import org.apache.hadoop.ozone.om.codec.OmVolumeArgsCodec; +import org.apache.hadoop.ozone.om.codec.OmPrefixInfoCodec; import org.apache.hadoop.ozone.om.codec.S3SecretValueCodec; import org.apache.hadoop.ozone.om.codec.TokenIdentifierCodec; import org.apache.hadoop.ozone.om.codec.VolumeListCodec; @@ -41,6 +42,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList; @@ -98,6 +100,8 @@ public class OmMetadataManagerImpl implements OMMetadataManager { * |-------------------------------------------------------------------| * | dTokenTable | s3g_access_key_id -> s3Secret | * |-------------------------------------------------------------------| + * | prefixInfoTable | prefix -> PrefixInfo | + * |-------------------------------------------------------------------| */ public static final String USER_TABLE = "userTable"; @@ -110,6 +114,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager { public static final String MULTIPARTINFO_TABLE = "multipartInfoTable"; public static final String S3_SECRET_TABLE = "s3SecretTable"; public static final String DELEGATION_TOKEN_TABLE = "dTokenTable"; + public static final String PREFIX_TABLE = "prefixTable"; private DBStore store; @@ -126,6 +131,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager { private Table multipartInfoTable; private Table s3SecretTable; private Table dTokenTable; + private Table prefixTable; public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException { this.lock = new OzoneManagerLock(conf); @@ -183,6 +189,11 @@ public Table getS3Table() { return s3Table; } + @Override + public Table getPrefixTable() { + return prefixTable; + } + @Override public Table getMultipartInfoTable() { return multipartInfoTable; @@ -230,13 +241,15 @@ protected DBStoreBuilder addOMTablesAndCodecs(DBStoreBuilder builder) { .addTable(MULTIPARTINFO_TABLE) .addTable(DELEGATION_TOKEN_TABLE) .addTable(S3_SECRET_TABLE) + .addTable(PREFIX_TABLE) .addCodec(OzoneTokenIdentifier.class, new TokenIdentifierCodec()) .addCodec(OmKeyInfo.class, new OmKeyInfoCodec()) .addCodec(OmBucketInfo.class, new OmBucketInfoCodec()) .addCodec(OmVolumeArgs.class, new OmVolumeArgsCodec()) .addCodec(VolumeList.class, new VolumeListCodec()) .addCodec(OmMultipartKeyInfo.class, new OmMultipartKeyInfoCodec()) - .addCodec(S3SecretValue.class, new S3SecretValueCodec()); + .addCodec(S3SecretValue.class, new S3SecretValueCodec()) + .addCodec(OmPrefixInfo.class, new OmPrefixInfoCodec()); } /** @@ -282,6 +295,10 @@ protected void initializeOmTables() throws IOException { s3SecretTable = this.store.getTable(S3_SECRET_TABLE, String.class, S3SecretValue.class); checkTableStatus(s3SecretTable, S3_SECRET_TABLE); + + prefixTable = this.store.getTable(PREFIX_TABLE, String.class, + OmPrefixInfo.class); + checkTableStatus(prefixTable, PREFIX_TABLE); } /**