From 53120e2e6cac3123581c6a749acc505f38c9a306 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Tue, 6 Nov 2018 16:53:04 -0800 Subject: [PATCH] HDDS-592. Fix ozone-secure.robot test. Contributed by Ajay Kumar. --- .../acceptance/ozone-secure.robot | 95 --------------- .../dist/src/main/compose/ozonesecure/.env | 1 - .../compose/ozonesecure/docker-compose.yaml | 22 ++-- .../docker-image/runner/Dockerfile | 4 +- .../smoketest/security/ozone-secure.robot | 111 ++++++++++++++++++ hadoop-ozone/dist/src/main/smoketest/test.sh | 2 + 6 files changed, 126 insertions(+), 109 deletions(-) delete mode 100644 hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot deleted file mode 100644 index 7fc1088e68..0000000000 --- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot +++ /dev/null @@ -1,95 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Smoke test to start cluster with docker-compose environments. -Library OperatingSystem -Suite Setup Startup Ozone Cluster -Suite Teardown Teardown Ozone Cluster - -*** Variables *** -${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root" -${version} - -*** Test Cases *** - -Daemons are running - Is daemon running om - Is daemon running scm - Is daemon running datanode - Is daemon running ozone.kdc - -Check if datanode is connected to the scm - Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 1 - -Test rest interface - ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1" - Should contain ${result} 201 Created - ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1" - Should contain ${result} 201 Created - ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1" - Should contain ${result} 200 OK - ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1" - Should contain ${result} 200 OK - -Test ozone cli - ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root - Should contain ${result} Client cannot authenticate via - # Authenticate testuser - Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab - Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root - ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")' - Should contain ${result} createdOn - Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB - ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name' - Should Be Equal ${result} bill - -*** Keywords *** - -Startup Ozone Cluster - ${rc} ${output} = Run docker compose 0 down - ${rc} ${output} = Run docker compose 0 up -d - Should Be Equal As Integers ${rc} 0 - Wait Until Keyword Succeeds 3min 10sec Is Daemon started ksm KSM is listening - -Teardown Ozone Cluster - Run docker compose 0 down - -Is daemon running - [arguments] ${name} - ${result} = Run docker ps - Should contain ${result} _${name}_1 - -Is Daemon started - [arguments] ${name} ${expression} - ${rc} ${result} = Run docker compose 0 logs - Should contain ${result} ${expression} - -Have healthy datanodes - [arguments] ${requirednodes} - ${result} = Execute on 0 scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value' - Should Be Equal ${result} ${requirednodes} - -Execute on - [arguments] ${expected_rc} ${componentname} ${command} - ${rc} ${return} = Run docker compose ${expected_rc} exec ${componentname} ${command} - [return] ${return} - -Run docker compose - [arguments] ${expected_rc} ${command} - Set Environment Variable OZONEDIR ${basedir}/hadoop-dist/target/ozone - ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command} - Should Be Equal As Integers ${rc} ${expected_rc} - [return] ${rc} ${output} diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env index a494004af2..cac418ae59 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env @@ -15,4 +15,3 @@ # limitations under the License. HDDS_VERSION=${hdds.version} -SRC_VOLUME=../../ diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml index fab5ba939a..034d0ffc0f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml @@ -24,7 +24,7 @@ services: buildno: 1 hostname: kdc volumes: - - $SRC_VOLUME:/opt/hadoop + - ../..:/opt/hadoop datanode: build: context: docker-image/runner @@ -32,13 +32,13 @@ services: args: buildno: 1 volumes: - - $SRC_VOLUME:/opt/hadoop + - ../..:/opt/hadoop hostname: datanode ports: - - 9864 + - 9864 command: ["/opt/hadoop/bin/ozone","datanode"] env_file: - - docker-config + - docker-config om: build: context: docker-image/runner @@ -47,13 +47,13 @@ services: buildno: 1 hostname: om volumes: - - $SRC_VOLUME:/opt/hadoop + - ../..:/opt/hadoop ports: - - 9874:9874 + - 9874:9874 environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION env_file: - - docker-config + - docker-config command: ["/opt/hadoop/bin/ozone","om"] scm: build: @@ -63,11 +63,11 @@ services: buildno: 1 hostname: scm volumes: - - $SRC_VOLUME:/opt/hadoop + - ../..:/opt/hadoop ports: - - 9876:9876 + - 9876:9876 env_file: - - docker-config + - docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["/opt/hadoop/bin/ozone","scm"] + command: ["/opt/hadoop/bin/ozone","scm"] \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile index efda03e08d..17eff186d0 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile @@ -15,8 +15,8 @@ # limitations under the License. FROM openjdk:8-jdk -RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean - +RUN apt-get update && apt-get install -y jq curl python python-pip sudo && apt-get clean +RUN pip install robotframework RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 RUN chmod +x /usr/local/bin/dumb-init RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot new file mode 100644 index 0000000000..457ebaaa1d --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Smoke test to start cluster with docker-compose environments. +Library OperatingSystem +Resource ../commonlib.robot + +*** Test Cases *** +Create volume and bucket + ${rc} ${output} = Run And Return Rc And Output ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root + Should contain ${output} Client cannot authenticate via + # Authenticate testuser + Execute kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab + Execute ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root + Execute ozone sh volume create o3://om/fstest2 --user bilbo --quota 100TB --root + Execute ozone sh bucket create o3://om/fstest/bucket1 + Execute ozone sh bucket create o3://om/fstest/bucket2 + Execute ozone sh bucket create o3://om/fstest2/bucket3 + +Check volume from ozonefs + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/ + +Run ozoneFS tests + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} testdir/deep + Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/ + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} NOTICE.txt + + Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} PUTFILE.txt + + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/ + Should contain ${result} NOTICE.txt + Should contain ${result} PUTFILE.txt + + Execute ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} MOVED.TXT + Should not contain ${result} NOTICE.txt + + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1 + Execute ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} subdir1/NOTICE.txt + + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/ + Should contain ${result} NOTICE.txt + + Execute ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + Should not contain ${result} Failed + + Execute ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should not contain ${result} NOTICE.txt + + ${result} = Execute ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/ + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should not contain ${result} subdir1 + + Execute ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} TOUCHFILE.txt + + Execute ozone fs -rm -r o3fs://bucket1.fstest/testdir/ + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should not contain ${result} testdir + + Execute rm -Rf localdir1 + Execute mkdir localdir1 + Execute cp NOTICE.txt localdir1/LOCAL.txt + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1 + Execute ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/ + Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt + + ${result} = Execute ozone fs -ls -R o3fs://bucket1.fstest/testdir1/ + Should contain ${result} localdir1/LOCAL.txt + Should contain ${result} testdir1/NOTICE.txt + + Execute ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2 + Execute ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3 + Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/ + Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/ + Execute ozone sh key put o3://om/fstest/bucket1/KEY.txt NOTICE.txt + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/KEY.txt + Should contain ${result} KEY.txt + ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt + Should Be Equal As Integers ${rc} 1 + Should contain ${result} File exists + Execute rm -Rf GET.txt + Execute ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt + Execute ls -l GET.txt + ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3fs://abcde.pqrs/ + Should Be Equal As Integers ${rc} 1 + Should contain ${result} VOLUME_NOT_FOUND + diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh index 5819595c46..53d0876eb0 100755 --- a/hadoop-ozone/dist/src/main/smoketest/test.sh +++ b/hadoop-ozone/dist/src/main/smoketest/test.sh @@ -107,6 +107,8 @@ if [ "$RUN_ALL" = true ]; then # # We select the test suites and execute them on multiple type of clusters # + DEFAULT_TESTS=("security") + execute_tests ozonesecure "${DEFAULT_TESTS[@]}" DEFAULT_TESTS=("basic") execute_tests ozone "${DEFAULT_TESTS[@]}" TESTS=("ozonefs")