diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java index ac25db5a96..29132302dc 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java @@ -183,4 +183,34 @@ private HddsConfigKeys() { public static final String HDDS_GRPC_TLS_TEST_CERT = "hdds.grpc.tls" + ".test_cert"; public static final boolean HDDS_GRPC_TLS_TEST_CERT_DEFAULT = false; + + // Comma separated acls (users, groups) allowing clients accessing + // datanode container protocol + // when hadoop.security.authorization is true, this needs to be set in + // hadoop-policy.xml, "*" allows all users/groups to access. + public static final String + HDDS_SECURITY_CLIENT_DATANODE_CONTAINER_PROTOCOL_ACL = + "hdds.security.client.datanode.container.protocol.acl"; + + // Comma separated acls (users, groups) allowing clients accessing + // scm container protocol + // when hadoop.security.authorization is true, this needs to be set in + // hadoop-policy.xml, "*" allows all users/groups to access. + public static final String HDDS_SECURITY_CLIENT_SCM_CONTAINER_PROTOCOL_ACL = + "hdds.security.client.scm.container.protocol.acl"; + + // Comma separated acls (users, groups) allowing clients accessing + // scm block protocol + // when hadoop.security.authorization is true, this needs to be set in + // hadoop-policy.xml, "*" allows all users/groups to access. + public static final String HDDS_SECURITY_CLIENT_SCM_BLOCK_PROTOCOL_ACL = + "hdds.security.client.scm.block.protocol.acl"; + + // Comma separated acls (users, groups) allowing clients accessing + // scm certificate protocol + // when hadoop.security.authorization is true, this needs to be set in + // hadoop-policy.xml, "*" allows all users/groups to access. + public static final String HDDS_SECURITY_CLIENT_SCM_CERTIFICATE_PROTOCOL_ACL = + "hdds.security.client.scm.certificate.protocol.acl"; + } \ No newline at end of file diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java index 696836a867..4036cb17b8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java @@ -31,6 +31,12 @@ @InterfaceAudience.Private public interface SCMSecurityProtocol { + @SuppressWarnings("checkstyle:ConstantName") + /** + * Version 1: Initial version. + */ + long versionID = 1L; + /** * Get SCM signed certificate for DataNode. * diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java index cc60c0aa56..41b0332d6d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java @@ -26,7 +26,7 @@ */ @ProtocolInfo(protocolName = - "org.apache.hadoop.ozone.protocol.SCMSecurityProtocol", + "org.apache.hadoop.hdds.protocol.SCMSecurityProtocol", protocolVersion = 1) @KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface SCMSecurityProtocolPB extends diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java index 88d06a83fd..6e95e7027c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java @@ -37,6 +37,12 @@ @KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface ScmBlockLocationProtocol extends Closeable { + @SuppressWarnings("checkstyle:ConstantName") + /** + * Version 1: Initial version. + */ + long versionID = 1L; + /** * Asks SCM where a block should be allocated. SCM responds with the * set of datanodes that should be used creating this block. diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 4e85fbe0e0..8831d16182 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -37,6 +37,13 @@ */ @KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface StorageContainerLocationProtocol extends Closeable { + + @SuppressWarnings("checkstyle:ConstantName") + /** + * Version 1: Initial version. + */ + long versionID = 1L; + /** * Asks SCM where a container should be allocated. SCM responds with the * set of datanodes that should be used creating this container. diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java index f80ba2010c..f0af7aaed8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java @@ -30,7 +30,7 @@ * Protocol Buffers service interface to add Hadoop-specific annotations. */ @ProtocolInfo(protocolName = - "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol", + "org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol", protocolVersion = 1) @KerberosInfo( serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 99f4e35c57..c6834e6ca8 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1963,5 +1963,50 @@ Keytab used by Freon. + + hdds.security.client.datanode.container.protocol.acl + * + SECURITY + + Comma separated list of users and groups allowed to access + client datanode container protocol. + + + + hdds.security.client.scm.block.protocol.acl + * + SECURITY + + Comma separated list of users and groups allowed to access + client scm block protocol. + + + + hdds.security.client.scm.certificate.protocol.acl + * + SECURITY + + Comma separated list of users and groups allowed to access + client scm certificate protocol. + + + + hdds.security.client.scm.container.protocol.acl + * + SECURITY + + Comma separated list of users and groups allowed to access + client scm container protocol. + + + + ozone.om.security.client.protocol.acl + * + SECURITY + + Comma separated list of users and groups allowed to access + client ozone manager protocol. + + diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java index 360058110e..61bdb27f4c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java @@ -47,6 +47,13 @@ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) @InterfaceAudience.Private public interface StorageContainerDatanodeProtocol { + + @SuppressWarnings("checkstyle:ConstantName") + /** + * Version 1: Initial version. + */ + long versionID = 1L; + /** * Returns SCM version. * @return Version info. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 75932b5b69..1d7b59d8ac 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -23,6 +23,7 @@ import com.google.common.collect.Maps; import com.google.protobuf.BlockingService; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos; @@ -117,7 +118,10 @@ public SCMBlockProtocolServer(OzoneConfiguration conf, updateRPCListenAddress( conf, OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, scmBlockAddress, blockRpcServer); - + if (conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, + false)) { + blockRpcServer.refreshServiceAcl(conf, SCMPolicyProvider.getInstance()); + } } public RPC.Server getBlockRpcServer() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 25ea3bc529..2b1022bbd3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.protobuf.BlockingService; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -133,7 +134,10 @@ public SCMClientProtocolServer(OzoneConfiguration conf, clientRpcAddress = updateRPCListenAddress(conf, OZONE_SCM_CLIENT_ADDRESS_KEY, scmAddress, clientRpcServer); - + if (conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, + false)) { + clientRpcServer.refreshServiceAcl(conf, SCMPolicyProvider.getInstance()); + } } public RPC.Server getClientRpcServer() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java index 3030aa7017..f07db622cd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.protobuf.BlockingService; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -179,6 +180,11 @@ public SCMDatanodeProtocolServer(final OzoneConfiguration conf, conf, OZONE_SCM_DATANODE_ADDRESS_KEY, datanodeRpcAddr, datanodeRpcServer); + if (conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, + false)) { + datanodeRpcServer.refreshServiceAcl(conf, + SCMPolicyProvider.getInstance()); + } } public void start() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMPolicyProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMPolicyProvider.java new file mode 100644 index 0000000000..b21a7222ac --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMPolicyProvider.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.server; + + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; +import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; +import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol; +import org.apache.hadoop.security.authorize.PolicyProvider; +import org.apache.hadoop.security.authorize.Service; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.hadoop.hdds.HddsConfigKeys.*; + +/** + * {@link PolicyProvider} for SCM protocols. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public final class SCMPolicyProvider extends PolicyProvider { + + private static AtomicReference atomicReference = + new AtomicReference<>(); + + private SCMPolicyProvider() { + } + + @Private + @Unstable + public static SCMPolicyProvider getInstance() { + if (atomicReference.get() == null) { + atomicReference.compareAndSet(null, new SCMPolicyProvider()); + } + return atomicReference.get(); + } + + private static final Service[] SCM_SERVICES = + new Service[]{ + new Service( + HDDS_SECURITY_CLIENT_DATANODE_CONTAINER_PROTOCOL_ACL, + StorageContainerDatanodeProtocol.class), + new Service( + HDDS_SECURITY_CLIENT_SCM_CONTAINER_PROTOCOL_ACL, + StorageContainerLocationProtocol.class), + new Service( + HDDS_SECURITY_CLIENT_SCM_BLOCK_PROTOCOL_ACL, + ScmBlockLocationProtocol.class), + new Service( + HDDS_SECURITY_CLIENT_SCM_CERTIFICATE_PROTOCOL_ACL, + SCMSecurityProtocol.class), + }; + + @SuppressFBWarnings("EI_EXPOSE_REP") + @Override + public Service[] getServices() { + return SCM_SERVICES; + } + +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java index 86bcbccbd2..05a1e04466 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java @@ -26,6 +26,7 @@ import java.util.concurrent.Future; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto; @@ -86,6 +87,10 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol { SCMSecurityProtocolPB.class, secureProtoPbService, handlerCount); + if (conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, + false)) { + rpcServer.refreshServiceAcl(conf, SCMPolicyProvider.getInstance()); + } } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index a0c58eac40..ab251cbeec 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -210,4 +210,10 @@ private OMConfigKeys() { public static final long OZONE_DB_CHECKPOINT_TRANSFER_RATE_DEFAULT = 0; //no throttling + // Comma separated acls (users, groups) allowing clients accessing + // OM client protocol + // when hadoop.security.authorization is true, this needs to be set in + // hadoop-policy.xml, "*" allows all users/groups to access. + public static final String OZONE_OM_SECURITY_CLIENT_PROTOCOL_ACL = + "ozone.om.security.client.protocol.acl"; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 2b9da1585e..6834043a90 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -47,6 +47,12 @@ public interface OzoneManagerProtocol extends OzoneManagerSecurityProtocol, Closeable { + @SuppressWarnings("checkstyle:ConstantName") + /** + * Version 1: Initial version. + */ + long versionID = 1L; + /** * Creates a volume. * @param args - Arguments to create Volume. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java index 175527b482..69083dc885 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java @@ -30,7 +30,7 @@ * Protocol used to communicate with OM. */ @ProtocolInfo(protocolName = - "org.apache.hadoop.ozone.protocol.OzoneManagerProtocol", + "org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol", protocolVersion = 1) @KerberosInfo( serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index bf8eda9018..b49396bd89 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -45,6 +45,13 @@ CORE-SITE.XML_hadoop.security.authentication=kerberos CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/ CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms +CORE-SITE.XML_hadoop.security.authorization=true +HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=* + HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBCSID.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBCSID.java index 19451cda2b..78023ef15c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBCSID.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBCSID.java @@ -55,8 +55,8 @@ */ public class TestBCSID { + private static OzoneConfiguration conf = new OzoneConfiguration(); private static MiniOzoneCluster cluster; - private static OzoneConfiguration conf; private static OzoneClient client; private static ObjectStore objectStore; private static String volumeName; @@ -69,7 +69,6 @@ public class TestBCSID { */ @BeforeClass public static void init() throws Exception { - conf = new OzoneConfiguration(); String path = GenericTestUtils .getTempPath(TestBCSID.class.getSimpleName()); File baseDir = new File(path); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java index 8740eba2d3..2415335290 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java @@ -70,7 +70,7 @@ public class TestCloseContainerHandlingByClient { private static MiniOzoneCluster cluster; - private static OzoneConfiguration conf; + private static OzoneConfiguration conf = new OzoneConfiguration(); private static OzoneClient client; private static ObjectStore objectStore; private static int chunkSize; @@ -88,7 +88,6 @@ public class TestCloseContainerHandlingByClient { */ @BeforeClass public static void init() throws Exception { - conf = new OzoneConfiguration(); chunkSize = (int) OzoneConsts.MB; blockSize = 4 * chunkSize; conf.set(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT, "5000ms"); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java index 5ac8c8e4d5..2c3cfab045 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java @@ -56,7 +56,7 @@ public class TestContainerStateMachine { private static MiniOzoneCluster cluster; - private static OzoneConfiguration conf; + private static OzoneConfiguration conf = new OzoneConfiguration(); private static OzoneClient client; private static ObjectStore objectStore; private static String volumeName; @@ -70,7 +70,6 @@ public class TestContainerStateMachine { */ @BeforeClass public static void init() throws Exception { - conf = new OzoneConfiguration(); path = GenericTestUtils .getTempPath(TestContainerStateMachine.class.getSimpleName()); File baseDir = new File(path); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 95c5048326..32792ae149 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -706,6 +706,7 @@ public void testPutKeyRatisThreeNodes() } + @Ignore("Debug Jenkins Timeout") @Test public void testPutKeyRatisThreeNodesParallel() throws IOException, InterruptedException { diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 9bcf6ff48d..143a6622a2 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -56,6 +56,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> 2.2.0 test + + com.google.code.findbugs + findbugs + 3.0.1 + compile + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPolicyProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPolicyProvider.java new file mode 100644 index 0000000000..67c7eb8bbc --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPolicyProvider.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.security.authorize.PolicyProvider; +import org.apache.hadoop.security.authorize.Service; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.hadoop.ozone.om.OMConfigKeys + .OZONE_OM_SECURITY_CLIENT_PROTOCOL_ACL; + +/** + * {@link PolicyProvider} for OM protocols. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public final class OMPolicyProvider extends PolicyProvider { + + private static AtomicReference atomicReference = + new AtomicReference<>(); + + private OMPolicyProvider() { + } + + @Private + @Unstable + public static OMPolicyProvider getInstance() { + if (atomicReference.get() == null) { + atomicReference.compareAndSet(null, new OMPolicyProvider()); + } + return atomicReference.get(); + } + + private static final Service[] OM_SERVICES = + new Service[]{ + new Service(OZONE_OM_SECURITY_CLIENT_PROTOCOL_ACL, + OzoneManagerProtocol.class), + }; + + @SuppressFBWarnings("EI_EXPOSE_REP") + @Override + public Service[] getServices() { + return OM_SERVICES; + } + +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 440257bde6..fdce64a1ad 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -263,20 +264,19 @@ private OzoneManager(OzoneConfiguration conf) throws IOException, // Load HA related configurations loadOMHAConfigs(configuration); - if (!testSecureOmFlag || !isOzoneSecurityEnabled()) { - scmContainerClient = getScmContainerClient(configuration); - // verifies that the SCM info in the OM Version file is correct. - scmBlockClient = getScmBlockClient(configuration); + scmContainerClient = getScmContainerClient(configuration); + // verifies that the SCM info in the OM Version file is correct. + scmBlockClient = getScmBlockClient(configuration); + + // For testing purpose only, not hit scm from om as Hadoop UGI can't login + // two principals in the same JVM. + if (!testSecureOmFlag) { ScmInfo scmInfo = scmBlockClient.getScmInfo(); if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo .getScmId().equals(omStorage.getScmId()))) { throw new OMException("SCM version info mismatch.", ResultCodes.SCM_VERSION_MISMATCH_ERROR); } - } else { - // For testing purpose only - scmContainerClient = null; - scmBlockClient = null; } RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class, @@ -778,6 +778,11 @@ private static RPC.Server startRpcServer(OzoneConfiguration conf, .build(); DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer); + + if (conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, + false)) { + rpcServer.refreshServiceAcl(conf, OMPolicyProvider.getInstance()); + } return rpcServer; }