From f267917ce3cf282b32166e39af871a8d1231d090 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Thu, 10 Oct 2019 22:33:59 -0700 Subject: [PATCH] HDDS-2282. scmcli pipeline list command throws NullPointerException. Contributed by Xiaoyu Yao. (#1642) --- .../hadoop/hdds/scm/XceiverClientManager.java | 4 ++- .../apache/hadoop/hdds/scm/cli/SCMCLI.java | 21 ++++++++++++-- .../dist/src/main/compose/ozonesecure/test.sh | 2 ++ .../src/main/smoketest/scmcli/pipeline.robot | 28 +++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java index 8218c77087..b15828a153 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java @@ -78,7 +78,9 @@ public class XceiverClientManager implements Closeable { private boolean isSecurityEnabled; private final boolean topologyAwareRead; /** - * Creates a new XceiverClientManager. + * Creates a new XceiverClientManager for non secured ozone cluster. + * For security enabled ozone cluster, client should use the other constructor + * with a valid ca certificate in pem string format. * * @param conf configuration */ diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java index ff30eca470..0b5c18e820 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands; @@ -36,17 +37,20 @@ import org.apache.hadoop.hdds.scm.protocolPB .StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.commons.lang3.StringUtils; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; +import static org.apache.hadoop.hdds.HddsUtils.getScmSecurityClient; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_CLIENT_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE; @@ -136,8 +140,21 @@ public ScmClient createScmClient() NetUtils.getDefaultSocketFactory(ozoneConf), Client.getRpcTimeout(ozoneConf))), StorageContainerLocationProtocol.class, ozoneConf); - return new ContainerOperationClient( - client, new XceiverClientManager(ozoneConf)); + + XceiverClientManager xceiverClientManager = null; + if (OzoneSecurityUtil.isSecurityEnabled(ozoneConf)) { + SecurityConfig securityConfig = new SecurityConfig(ozoneConf); + SCMSecurityProtocol scmSecurityProtocolClient = getScmSecurityClient( + (OzoneConfiguration) securityConfig.getConfiguration()); + String caCertificate = + scmSecurityProtocolClient.getCACertificate(); + xceiverClientManager = new XceiverClientManager(ozoneConf, + OzoneConfiguration.of(ozoneConf).getObject(XceiverClientManager + .ScmClientConfig.class), caCertificate); + } else { + xceiverClientManager = new XceiverClientManager(ozoneConf); + } + return new ContainerOperationClient(client, xceiverClientManager); } public void checkContainerExists(ScmClient scmClient, long containerId) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh index 01106b8615..f32846386a 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -35,6 +35,8 @@ execute_robot_test scm ozonefs/ozonefs.robot execute_robot_test s3g s3 +execute_robot_test scm scmcli + stop_docker_env generate_report diff --git a/hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot b/hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot new file mode 100644 index 0000000000..6a6f0b0eb7 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Smoketest ozone cluster startup +Library OperatingSystem +Library BuiltIn +Resource ../commonlib.robot + +*** Variables *** + + +*** Test Cases *** +Run list pipeline + ${output} = Execute ozone scmcli pipeline list + Should contain ${output} Type:RATIS, Factor:ONE, State:OPEN \ No newline at end of file