big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-352. Separate install and testing phases in acceptance tests. Contributed by Elek Marton.

Browse Source
This commit is contained in:
Bharat Viswanadham 2018-09-17 17:13:52 -07:00
parent 0a26c521f0
commit 8b2f5e60fa
28 changed files with 250 additions and 618 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dev-support/bin/ozone-dist-layout-stitching
View File

@ -169,8 +169,9 @@ find . -name jdiff -type d | xargs rm -rf
#add ozone specific readme
run cp "${ROOT}/hadoop-dist/src/main/ozone/README.txt" README.txt
#Copy docker compose files
#Copy docker compose files and robot tests
run cp -p -r "${ROOT}/hadoop-dist/src/main/compose" .
run cp -p -r "${ROOT}/hadoop-dist/src/main/smoketest" .
mkdir -p ./share/hadoop/mapreduce
mkdir -p ./share/hadoop/yarn

1
hadoop-dist/src/main/compose/ozone-hdfs/docker-config
View File

@ -15,6 +15,7 @@
# limitations under the License.
OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=true
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id

1
hadoop-dist/src/main/compose/ozone/docker-config
View File

@ -15,6 +15,7 @@
# limitations under the License.
OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id

8
hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml → hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
View File

@ -19,7 +19,7 @@ services:
datanode:
image: apache/hadoop-runner
volumes:
- ${OZONEDIR}:/opt/hadoop
- ../..:/opt/hadoop
ports:
- 9864
command: ["/opt/hadoop/bin/ozone","datanode"]
@ -29,7 +29,7 @@ services:
image: apache/hadoop-runner
hostname: ozoneManager
volumes:
- ${OZONEDIR}:/opt/hadoop
- ../..:/opt/hadoop
ports:
- 9874
environment:
@ -40,7 +40,7 @@ services:
scm:
image: apache/hadoop-runner
volumes:
- ${OZONEDIR}:/opt/hadoop
- ../..:/opt/hadoop
ports:
- 9876
env_file:
@ -51,7 +51,7 @@ services:
hadooplast:
image: flokkr/hadoop:3.1.0
volumes:
- ${OZONEDIR}:/opt/ozone
- ../..:/opt/ozone
env_file:
- ./docker-config
environment:

0
hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config → hadoop-dist/src/main/compose/ozonefs/docker-config
View File

1
hadoop-dist/src/main/compose/ozoneperf/docker-config
View File

@ -15,6 +15,7 @@
# limitations under the License.
OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id

1
hadoop-dist/src/main/compose/ozonescripts/docker-config
View File

@ -21,6 +21,7 @@ OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=true
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm

30
hadoop-dist/src/main/smoketest/README.md Normal file
View File

@ -0,0 +1,30 @@
<!---
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
## Ozone Acceptance Tests
This directory contains a [robotframework](http://robotframework.org/) based test suite for Ozone to make it easier to check the current state of the package.
You can run in in any environment after [installing](https://github.com/robotframework/robotframework/blob/master/INSTALL.rst)
```
cd $DIRECTORY_OF_OZONE
robot smoketest/bascic
```
The argument of the `robot` could be any robot file or directory.
The current configuration in the robot files (hostnames, ports) are adjusted for the docker-based setup but you can easily modify it for any environment.
The `./test.sh` in this directory can start multiple type of clusters (ozone standalon or ozone + hdfs) and execute the test framework with all of the clusters.

27
hadoop-ozone/acceptance-test/src/test/acceptance/basic/basic.robot → hadoop-dist/src/main/smoketest/basic/basic.robot
View File

@ -16,35 +16,32 @@
*** Settings ***
Documentation Smoketest ozone cluster startup
Library OperatingSystem
Suite Setup Startup Ozone cluster with size 5
Suite Teardown Teardown Ozone cluster
Resource ../commonlib.robot
*** Variables ***
${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
${COMPOSEFILE} ${CURDIR}/docker-compose.yaml
${PROJECTDIR} ${CURDIR}/../../../../../..
${DATANODE_HOST} localhost
*** Test Cases ***
Test rest interface
${result} = Execute on datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"
Should contain ${result} 201 Created
${result} = Execute on datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"
Should contain ${result} 201 Created
${result} = Execute on datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"
Should contain ${result} 200 OK
${result} = Execute on datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"
Should contain ${result} 200 OK
Check webui static resources
${result} = Execute on scm curl -s -I http://localhost:9876/static/bootstrap-3.3.7/js/bootstrap.min.js
Should contain ${result} 200
${result} = Execute on ozoneManager curl -s -I http://localhost:9874/static/bootstrap-3.3.7/js/bootstrap.min.js
Should contain ${result} 200
${result} = Execute curl -s -I http://scm:9876/static/bootstrap-3.3.7/js/bootstrap.min.js
Should contain ${result} 200
${result} = Execute curl -s -I http://ozoneManager:9874/static/bootstrap-3.3.7/js/bootstrap.min.js
Should contain ${result} 200
Start freon testing
${result} = Execute on ozoneManager ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 10
Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125
Should Not Contain ${result} ERROR
${result} = Execute ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 10
Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125
Should Not Contain ${result} ERROR

82
hadoop-dist/src/main/smoketest/basic/ozone-shell.robot Normal file
View File

@ -0,0 +1,82 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
Documentation Test ozone shell CLI usage
Library OperatingSystem
Resource ../commonlib.robot
Test Timeout 2 minute
*** Variables ***
*** Test Cases ***
RestClient without http port
Test ozone shell http:// ozoneManager restwoport
RestClient with http port
Test ozone shell http:// ozoneManager:9874 restwport
RestClient without host name
Test ozone shell http:// ${EMPTY} restwohost
RpcClient with port
Test ozone shell o3:// ozoneManager:9862 rpcwoport
RpcClient without host
Test ozone shell o3:// ${EMPTY} rpcwport
RpcClient without scheme
Test ozone shell ${EMPTY} ${EMPTY} rpcwoscheme
*** Keywords ***
Test ozone shell
[arguments] ${protocol} ${server} ${volume}
${result} = Execute ozone sh volume create ${protocol}${server}/${volume} --user bilbo --quota 100TB --root
Should not contain ${result} Failed
Should contain ${result} Creating Volume: ${volume}
${result} = Execute ozone sh volume list ${protocol}${server}/ --user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="${volume}")'
Should contain ${result} createdOn
${result} = Execute ozone sh volume list --user bilbo | grep -Ev 'Removed|DEBUG|ERROR|INFO|TRACE|WARN' | jq -r '.[] | select(.volumeName=="${volume}")'
Should contain ${result} createdOn
Execute ozone sh volume update ${protocol}${server}/${volume} --user bill --quota 10TB
${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .owner | .name'
Should Be Equal ${result} bill
${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .quota | .size'
Should Be Equal ${result} 10
Execute ozone sh bucket create ${protocol}${server}/${volume}/bb1
${result} = Execute ozone sh bucket info ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .storageType'
Should Be Equal ${result} DISK
${result} = Execute ozone sh bucket update ${protocol}${server}/${volume}/bb1 --addAcl user:frodo:rw,group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="samwise") | .type'
Should Be Equal ${result} GROUP
${result} = Execute ozone sh bucket update ${protocol}${server}/${volume}/bb1 --removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type'
Should Be Equal ${result} USER
${result} = Execute ozone sh bucket list ${protocol}${server}/${volume}/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
Should Be Equal ${result} ${volume}
Run Keyword Test key handling ${protocol} ${server} ${volume}
Execute ozone sh bucket delete ${protocol}${server}/${volume}/bb1
Execute ozone sh volume delete ${protocol}${server}/${volume} --user bilbo
Test key handling
[arguments] ${protocol} ${server} ${volume}
Execute ozone sh key put ${protocol}${server}/${volume}/bb1/key1 /opt/hadoop/NOTICE.txt
Execute rm -f NOTICE.txt.1
Execute ozone sh key get ${protocol}${server}/${volume}/bb1/key1 NOTICE.txt.1
Execute ls -l NOTICE.txt.1
${result} = Execute ozone sh key info ${protocol}${server}/${volume}/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
Should contain ${result} createdOn
${result} = Execute ozone sh key list ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
Should Be Equal ${result} key1
Execute ozone sh key delete ${protocol}${server}/${volume}/bb1/key1

12
hadoop-ozone/acceptance-test/dev-support/bin/robot-all.sh → hadoop-dist/src/main/smoketest/commonlib.robot Executable file → Normal file
View File

@ -1,4 +1,3 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@ -14,5 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
"$DIR/robot.sh" "$DIR/../../src/test/acceptance"
*** Keywords ***
Execute
[arguments] ${command}
${rc} ${output} = Run And Return Rc And Output ${command}
Log ${output}
Should Be Equal As Integers ${rc} 0
[return] ${output}

16
hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot → hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot
View File

@ -16,24 +16,20 @@
*** Settings ***
Documentation Ozonefs test
Library OperatingSystem
Suite Setup Startup Ozone cluster with size 5
Suite Teardown Teardown Ozone cluster
Resource ../commonlib.robot
*** Variables ***
${COMPOSEFILE} ${CURDIR}/docker-compose.yaml
${PROJECTDIR} ${CURDIR}/../../../../../..
*** Test Cases ***
Create volume and bucket
Execute on datanode ozone sh volume create http://ozoneManager/fstest --user bilbo --quota 100TB --root
Execute on datanode ozone sh bucket create http://ozoneManager/fstest/bucket1
Execute ozone sh volume create http://ozoneManager/fstest --user bilbo --quota 100TB --root
Execute ozone sh bucket create http://ozoneManager/fstest/bucket1
Check volume from ozonefs
${result} = Execute on datanode ozone fs -ls o3://bucket1.fstest/
${result} = Execute ozone fs -ls o3://bucket1.fstest/
Create directory from ozonefs
Execute on datanode ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep
${result} = Execute on ozoneManager ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} testdir/deep
Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} testdir/deep

101
hadoop-dist/src/main/smoketest/test.sh Executable file
View File

@ -0,0 +1,101 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
execute_tests(){
COMPOSE_FILE=$DIR/../compose/$1/docker-compose.yaml
TESTS=$2
echo "Executing test ${TESTS[*]} with $COMPOSE_FILE"
docker-compose -f "$COMPOSE_FILE" down
docker-compose -f "$COMPOSE_FILE" up -d
docker-compose -f "$COMPOSE_FILE" exec datanode sudo apt-get update
docker-compose -f "$COMPOSE_FILE" exec datanode sudo apt-get install -y python-pip
docker-compose -f "$COMPOSE_FILE" exec datanode sudo pip install robotframework
for TEST in "${TESTS[@]}"; do
set +e
docker-compose -f "$COMPOSE_FILE" exec datanode python -m robot "smoketest/$TEST"
set -e
done
if [ "$KEEP_RUNNING" = false ]; then
docker-compose -f "$COMPOSE_FILE" down
fi
}
RUN_ALL=true
KEEP_RUNNING=false
POSITIONAL=()
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
--env)
DOCKERENV="$2"
RUN_ALL=false
shift # past argument
shift # past value
;;
--keep)
KEEP_RUNNING=true
shift # past argument
;;
--help|-h|-help)
cat << EOF
Acceptance test executor for ozone.
This is a lightweight test executor for ozone.
You can run it with
./test.sh
Which executes all the tests in all the available environments.
Or you can run manually one test with
./test.sh --keep --env ozone-hdfs basic
--keep means that docker cluster won't be stopped after the test (optional)
--env defines the subdirectory under the compose dir
The remaining parameters define the test suites under smoketest dir.
Could be any directory or robot file relative to the smoketest dir.
EOF
exit 0
;;
*)
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
if [ "$RUN_ALL" = true ]; then
#
# This is the definition of the ozone acceptance test suite
#
# We select the test suites and execute them on multiple type of clusters
#
DEFAULT_TESTS=("basic")
execute_tests ozone "${DEFAULT_TESTS[@]}"
TESTS=("ozonefs")
execute_tests ozonefs "${TESTS[@]}"
else
execute_tests "$DOCKERENV" "${POSITIONAL[@]}"
fi

48
hadoop-ozone/acceptance-test/README.md
View File

@ -1,48 +0,0 @@
<!---
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
# Acceptance test suite for Ozone/Hdds
This project contains acceptance tests for ozone/hdds using docker-compose and [robot framework](http://robotframework.org/).
## Run
To run the acceptance tests, please activate the `ozone-acceptance-test` profile and do a full build.
```
mvn clean install -Pdist -Phdds
cd hadoop-ozone/acceptance-test
mvn integration-test -Phdds,ozone-acceptance-test,dist -DskipTests
```
Notes:
1. You need a hadoop build in hadoop-dist/target directory.
2. The `ozone-acceptance-test` could be activated with profile even if the unit tests are disabled.
3. This method does not require the robot framework on path as jpython is used.
## Development
You can also run manually the robot tests with `robot` cli.
(See robotframework docs to install it: http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#installation-instructions)
In the dev-support directory we have two wrapper scripts to run robot framework with local robot cli
instead of calling it from maven.
It's useful during the development of the robot files as any robotframework cli
arguments could be used.
1. `dev-support/bin/robot.sh` is the simple wrapper. The .robot file should be used as an argument.
2. `dev-support/bin/robot-all.sh` will call the robot.sh with the main acceptance test directory,
which means all the acceptance tests will be executed.

57
hadoop-ozone/acceptance-test/dev-support/bin/robot-dnd-all.sh
View File

@ -1,57 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -x
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
#Dir to save the results
TARGETDIR="$DIR/../../target/dnd"
#Name and imagename of the temporary, dind based test containers
DOCKER_IMAGE_NAME=ozoneacceptance
DOCKER_INSTANCE_NAME="${DOCKER_INSTANCE_NAME:-ozoneacceptance}"
teardown() {
docker stop "$DOCKER_INSTANCE_NAME"
}
trap teardown EXIT
#Make sure it will work even if the ozone is built by an other user. We
# eneable to run the distribution by an other user
mkdir -p "$TARGETDIR"
mkdir -p "$OZONEDIST/logs"
chmod o+w "$OZONEDIST/logs" || true
chmod -R o+w "$OZONEDIST/etc/hadoop" || true
chmod o+w "$OZONEDIST" || true
rm "$TARGETDIR/docker-compose.log"
docker rm "$DOCKER_INSTANCE_NAME" || true
docker build -t "$DOCKER_IMAGE_NAME" "$DIR/../docker"
#Starting the dind based environment
docker run --rm -v "$DIR/../../../..:/opt/hadoop" --privileged -d --name "$DOCKER_INSTANCE_NAME" $DOCKER_IMAGE_NAME
sleep 5
#Starting the tests
docker exec "$DOCKER_INSTANCE_NAME" /opt/hadoop/hadoop-ozone/acceptance-test/dev-support/bin/robot-all.sh
RESULT=$?
docker cp "$DOCKER_INSTANCE_NAME:/root/log.html" "$TARGETDIR/"
docker cp "$DOCKER_INSTANCE_NAME:/root/junit-results.xml" "$TARGETDIR/"
docker cp "$DOCKER_INSTANCE_NAME:/root/docker-compose.log" "$TARGETDIR/"
exit $RESULT

38
hadoop-ozone/acceptance-test/dev-support/bin/robot.sh
View File

@ -1,38 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -x
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
if [ ! "$(command -v robot)" ] ; then
echo ""
echo "robot is not on your PATH."
echo ""
echo "Please install it according to the documentation:"
echo " http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#installation-instructions"
echo " (TLDR; most of the time you need: 'pip install robotframework')"
exit -1
fi
MARKERFILE=$(find "$DIR/../../../../hadoop-dist/target" -name hadoop-ozone.sh)
OZONEDISTDIR="$(dirname "$(dirname "$(dirname "$MARKERFILE")")")"
if [ ! -d "$OZONEDISTDIR" ]; then
echo "Ozone can't be found in the $OZONEDISTDIR."
echo "You may need a full build with -Phdds and -Pdist profiles"
exit -1
fi
robot --variable "OZONEDIR:$OZONEDISTDIR" -x junit-results.xml "$@"

21
hadoop-ozone/acceptance-test/dev-support/docker/Dockerfile
View File

@ -1,21 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM docker:18-dind
RUN apk add --update python3 bash curl jq sudo
RUN pip3 install robotframework docker-compose
WORKDIR /root
USER root

23
hadoop-ozone/acceptance-test/dev-support/docker/docker-compose.yaml
View File

@ -1,23 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: "3"
services:
robotenv:
build: .
privileged: true
volumes:
- ../../../..:/opt/hadoop

65
hadoop-ozone/acceptance-test/pom.xml
View File

@ -1,65 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone</artifactId>
<version>0.3.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-ozone-acceptance-test</artifactId>
<version>3.2.0-SNAPSHOT</version>
<description>Apache Hadoop Ozone Acceptance Tests</description>
<name>Apache Hadoop Ozone Acceptance Tests</name>
<packaging>pom</packaging>
<properties>
<hadoop.component>ozone</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
</properties>
<profiles>
<profile>
<id>ozone-acceptance-test</id>
<build>
<plugins>
<plugin>
<groupId>org.robotframework</groupId>
<artifactId>robotframework-maven-plugin</artifactId>
<version>1.4.7</version>
<executions>
<execution>
<goals>
<goal>run</goal>
</goals>
<configuration>
<testCasesDirectory>src/test/acceptance</testCasesDirectory>
<variables>
<variable>OZONEDIR:${project.basedir}/../../hadoop-dist/target/ozone-${ozone.version}</variable>
</variables>
<skip>false</skip>
<skipTests>false</skipTests>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

17
hadoop-ozone/acceptance-test/src/test/acceptance/basic/.env
View File

@ -1,17 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
OZONEDIR=../../../../../../hadoop-dist/target/ozone

50
hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
View File

@ -1,50 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: "3"
services:
datanode:
image: apache/hadoop-runner
volumes:
- ${OZONEDIR}:/opt/hadoop
ports:
- 9864
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- ./docker-config
ozoneManager:
image: apache/hadoop-runner
hostname: ozoneManager
volumes:
- ${OZONEDIR}:/opt/hadoop
ports:
- 9874
environment:
ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
env_file:
- ./docker-config
command: ["/opt/hadoop/bin/ozone","om"]
scm:
image: apache/hadoop-runner
volumes:
- ${OZONEDIR}:/opt/hadoop
ports:
- 9876
env_file:
- ./docker-config
environment:
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
command: ["/opt/hadoop/bin/ozone","scm"]

34
hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
View File

@ -1,34 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN

86
hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot
View File

@ -1,86 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
Documentation Test ozone shell CLI usage
Library OperatingSystem
Suite Setup Startup Ozone cluster with size 5
Suite Teardown Teardown Ozone cluster
Resource ../commonlib.robot
Test Timeout 2 minute
*** Variables ***
${COMPOSEFILE} ${CURDIR}/docker-compose.yaml
${PROJECTDIR} ${CURDIR}/../../../../../..
*** Test Cases ***
RestClient without http port
Test ozone shell http:// ozoneManager restwoport
RestClient with http port
Test ozone shell http:// ozoneManager:9874 restwport
RestClient without host name
Test ozone shell http:// ${EMPTY} restwohost
RpcClient with port
Test ozone shell o3:// ozoneManager:9862 rpcwoport
RpcClient without host
Test ozone shell o3:// ${EMPTY} rpcwport
RpcClient without scheme
Test ozone shell ${EMPTY} ${EMPTY} rpcwoscheme
*** Keywords ***
Test ozone shell
[arguments] ${protocol} ${server} ${volume}
${result} = Execute on datanode ozone sh volume create ${protocol}${server}/${volume} --user bilbo --quota 100TB --root
Should not contain ${result} Failed
Should contain ${result} Creating Volume: ${volume}
${result} = Execute on datanode ozone sh volume list ${protocol}${server}/ --user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="${volume}")'
Should contain ${result} createdOn
${result} = Execute on datanode ozone sh volume list --user bilbo | grep -Ev 'Removed|DEBUG|ERROR|INFO|TRACE|WARN' | jq -r '.[] | select(.volumeName=="${volume}")'
Should contain ${result} createdOn
Execute on datanode ozone sh volume update ${protocol}${server}/${volume} --user bill --quota 10TB
${result} = Execute on datanode ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .owner | .name'
Should Be Equal ${result} bill
${result} = Execute on datanode ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .quota | .size'
Should Be Equal ${result} 10
Execute on datanode ozone sh bucket create ${protocol}${server}/${volume}/bb1
${result} = Execute on datanode ozone sh bucket info ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .storageType'
Should Be Equal ${result} DISK
${result} = Execute on datanode ozone sh bucket update ${protocol}${server}/${volume}/bb1 --addAcl user:frodo:rw,group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="samwise") | .type'
Should Be Equal ${result} GROUP
${result} = Execute on datanode ozone sh bucket update ${protocol}${server}/${volume}/bb1 --removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type'
Should Be Equal ${result} USER
${result} = Execute on datanode ozone sh bucket list ${protocol}${server}/${volume}/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
Should Be Equal ${result} ${volume}
Run Keyword Test key handling ${protocol} ${server} ${volume}
Execute on datanode ozone sh bucket delete ${protocol}${server}/${volume}/bb1
Execute on datanode ozone sh volume delete ${protocol}${server}/${volume} --user bilbo
Test key handling
[arguments] ${protocol} ${server} ${volume}
Execute on datanode ozone sh key put ${protocol}${server}/${volume}/bb1/key1 NOTICE.txt
Execute on datanode rm -f NOTICE.txt.1
Execute on datanode ozone sh key get ${protocol}${server}/${volume}/bb1/key1 NOTICE.txt.1
Execute on datanode ls -l NOTICE.txt.1
${result} = Execute on datanode ozone sh key info ${protocol}${server}/${volume}/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
Should contain ${result} createdOn
${result} = Execute on datanode ozone sh key list ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
Should Be Equal ${result} key1
Execute on datanode ozone sh key delete ${protocol}${server}/${volume}/bb1/key1

78
hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
View File

@ -1,78 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Keywords ***
Startup Ozone cluster with size
[arguments] ${datanodeno}
${rc} ${output} = Run docker compose down
Run echo "Starting new docker-compose environment" >> docker-compose.log
${rc} ${output} = Run docker compose up -d
Should Be Equal As Integers ${rc} 0
Wait Until Keyword Succeeds 1min 5sec Is Daemon started ozoneManager HTTP server of OZONEMANAGER is listening
Daemons are running without error
Scale datanodes up 5
Daemons are running without error
Is daemon running without error ozoneManager
Is daemon running without error scm
Is daemon running without error datanode
Check if datanode is connected to the scm
Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 1
Scale it up to 5 datanodes
Scale datanodes up 5
Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 5
Scale datanodes up
[arguments] ${datanodeno}
Run docker compose scale datanode=${datanodeno}
Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes ${datanodeno}
Teardown Ozone cluster
Run docker compose down
Run docker compose logs >> docker-compose.log
Is daemon running without error
[arguments] ${name}
${result} = Run docker ps
Should contain ${result} _${name}_1
${rc} ${result} = Run docker compose logs ${name}
Should not contain ${result} ERROR
Is Daemon started
[arguments] ${name} ${expression}
${rc} ${result} = Run docker compose logs
Should contain ${result} ${expression}
Have healthy datanodes
[arguments] ${requirednodes}
${result} = Execute on scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
Should Be Equal ${result} ${requirednodes}
Execute on
[arguments] ${componentname} ${command}
${rc} ${return} = Run docker compose exec -T ${componentname} ${command}
[return] ${return}
Run docker compose
[arguments] ${command}
Set Environment Variable COMPOSE_INTERACTIVE_NO_CLI 1
Set Environment Variable OZONEDIR ${OZONEDIR}
${rc} ${output} = Run And Return Rc And Output docker-compose -f ${COMPOSEFILE} ${command}
Log ${output}
Should Be Equal As Integers ${rc} 0
[return] ${rc} ${output}

17
hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/.env
View File

@ -1,17 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
OZONEDIR=../../../../../../hadoop-dist/target/ozone

49
hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot
View File

@ -1,49 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
Documentation Ozone Single Node Test
Library OperatingSystem
Suite Setup Startup Ozone cluster with size 1
Suite Teardown Teardown Ozone cluster
Resource ../commonlib.robot
*** Variables ***
${COMPOSEFILE} ${CURDIR}/docker-compose.yaml
${PROJECTDIR} ${CURDIR}/../../../../../..
*** Test Cases ***
Create volume and bucket
Execute on datanode ozone sh volume create http://ozoneManager/fstest --user bilbo --quota 100TB --root
Execute on datanode ozone sh bucket create http://ozoneManager/fstest/bucket1
Check volume from ozonefs
${result} = Execute on datanode ozone fs -ls o3://bucket1.fstest/
Create directory from ozonefs
Execute on datanode ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep
${result} = Execute on ozoneManager ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} testdir/deep
Test key handling
Execute on datanode ozone sh key put o3://ozoneManager/fstest/bucket1/key1 NOTICE.txt --replication ONE
Execute on datanode rm -f NOTICE.txt.1
Execute on datanode ozone sh key get o3://ozoneManager/fstest/bucket1/key1 NOTICE.txt.1
Execute on datanode ls -l NOTICE.txt.1
${result} = Execute on datanode ozone sh key info o3://ozoneManager/fstest/bucket1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
Should contain ${result} createdOn
${result} = Execute on datanode ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
Should Be Equal ${result} key1
Execute on datanode ozone sh key delete o3://ozoneManager/fstest/bucket1/key1

1
hadoop-ozone/pom.xml
View File

@ -121,7 +121,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<configuration>
<excludes>
<exclude>**/target/**</exclude>
<exclude>acceptance-test/hadoop-ozone-acceptance-test.iml</exclude>
<exclude>.gitattributes</exclude>
<exclude>.idea/**</exclude>
<exclude>dev-support/checkstyle*</exclude>

1
pom.xml
View File

@ -745,7 +745,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<modules>
<module>hadoop-ozone</module>
<module>hadoop-hdds</module>
<module>hadoop-ozone/acceptance-test</module>
</modules>
</profile>
</profiles>