parent
49e1292ea3
commit
1a696cc761
@ -122,3 +122,6 @@ run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" .
|
||||
run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/blockade" .
|
||||
run cp -p -r "${ROOT}/hadoop-ozone/dist/target/k8s" kubernetes
|
||||
run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/Dockerfile" .
|
||||
|
||||
#workaround for https://issues.apache.org/jira/browse/MRESOURCES-236
|
||||
find ./compose -name "*.sh" -exec chmod 755 {} \;
|
||||
|
2
hadoop-ozone/dist/pom.xml
vendored
2
hadoop-ozone/dist/pom.xml
vendored
@ -140,7 +140,7 @@
|
||||
<version>3.1.0</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-compose</id>
|
||||
<id>copy-compose-files</id>
|
||||
<phase>compile</phase>
|
||||
<goals>
|
||||
<goal>copy-resources</goal>
|
||||
|
35
hadoop-ozone/dist/src/main/compose/ozone/test.sh
vendored
Executable file
35
hadoop-ozone/dist/src/main/compose/ozone/test.sh
vendored
Executable file
@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
export COMPOSE_DIR
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$COMPOSE_DIR/../testlib.sh"
|
||||
|
||||
start_docker_env
|
||||
|
||||
#Due to the limitation of the current auditparser test, it should be the
|
||||
#first test in a clean cluster.
|
||||
|
||||
execute_robot_test scm auditparser
|
||||
|
||||
execute_robot_test scm basic/basic.robot
|
||||
|
||||
stop_docker_env
|
||||
|
||||
generate_report
|
39
hadoop-ozone/dist/src/main/compose/ozonefs/test.sh
vendored
Executable file
39
hadoop-ozone/dist/src/main/compose/ozonefs/test.sh
vendored
Executable file
@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
export COMPOSE_DIR
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$COMPOSE_DIR/../testlib.sh"
|
||||
|
||||
start_docker_env
|
||||
|
||||
execute_robot_test scm ozonefs/ozonefs.robot
|
||||
|
||||
|
||||
## TODO: As of now the o3fs tests are unstable.
|
||||
|
||||
export OZONE_HOME=/opt/ozone
|
||||
|
||||
#execute_robot_test hadoop32 ozonefs/hadoopo3fs.robot
|
||||
|
||||
#execute_robot_test hadoop31 ozonefs/hadoopo3fs.robot
|
||||
|
||||
stop_docker_env
|
||||
|
||||
generate_report
|
30
hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh
vendored
Executable file
30
hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh
vendored
Executable file
@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
export COMPOSE_DIR
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$COMPOSE_DIR/../testlib.sh"
|
||||
|
||||
start_docker_env
|
||||
|
||||
execute_robot_test scm basic/basic.robot
|
||||
|
||||
stop_docker_env
|
||||
|
||||
generate_report
|
32
hadoop-ozone/dist/src/main/compose/ozones3/test.sh
vendored
Executable file
32
hadoop-ozone/dist/src/main/compose/ozones3/test.sh
vendored
Executable file
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
export COMPOSE_DIR
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$COMPOSE_DIR/../testlib.sh"
|
||||
|
||||
start_docker_env
|
||||
|
||||
execute_robot_test scm basic/basic.robot
|
||||
|
||||
execute_robot_test scm s3
|
||||
|
||||
stop_docker_env
|
||||
|
||||
generate_report
|
40
hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
vendored
Executable file
40
hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
vendored
Executable file
@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
export COMPOSE_DIR
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$COMPOSE_DIR/../testlib.sh"
|
||||
|
||||
export SECURITY_ENABLED=true
|
||||
|
||||
start_docker_env
|
||||
|
||||
execute_robot_test scm kinit.robot
|
||||
|
||||
execute_robot_test scm basic
|
||||
|
||||
execute_robot_test scm security
|
||||
|
||||
execute_robot_test scm ozonefs/ozonefs.robot
|
||||
|
||||
execute_robot_test scm s3
|
||||
|
||||
stop_docker_env
|
||||
|
||||
generate_report
|
47
hadoop-ozone/dist/src/main/compose/test-all.sh
vendored
Executable file
47
hadoop-ozone/dist/src/main/compose/test-all.sh
vendored
Executable file
@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
#
|
||||
# Test executor to test all the compose/*/test.sh test scripts.
|
||||
#
|
||||
|
||||
SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
|
||||
ALL_RESULT_DIR="$SCRIPT_DIR/result"
|
||||
|
||||
mkdir -p "$ALL_RESULT_DIR"
|
||||
rm "$ALL_RESULT_DIR/*"
|
||||
|
||||
RESULT=0
|
||||
IFS=$'\n'
|
||||
# shellcheck disable=SC2044
|
||||
for test in $(find $SCRIPT_DIR -name test.sh); do
|
||||
echo "Executing test in $(dirname "$test")"
|
||||
|
||||
#required to read the .env file from the right location
|
||||
cd "$(dirname "$test")" || continue
|
||||
$test
|
||||
ret=$?
|
||||
if [[ $ret -ne 0 ]]; then
|
||||
RESULT=-1
|
||||
fi
|
||||
RESULT_DIR="$(dirname "$test")/result"
|
||||
cp "$RESULT_DIR"/robot-*.xml "$ALL_RESULT_DIR"
|
||||
done
|
||||
|
||||
docker run --rm -v "$SCRIPT_DIR/result:/opt/result" apache/hadoop-runner rebot -N "smoketests" -d "/opt/result" "/opt/result/robot-*.xml"
|
||||
exit $RESULT
|
53
hadoop-ozone/dist/src/main/compose/test-single.sh
vendored
Executable file
53
hadoop-ozone/dist/src/main/compose/test-single.sh
vendored
Executable file
@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#
|
||||
# Single test executor, can start a single robot test in any running container.
|
||||
#
|
||||
|
||||
|
||||
COMPOSE_DIR="$PWD"
|
||||
export COMPOSE_DIR
|
||||
|
||||
if [[ ! -f "$COMPOSE_DIR/docker-compose.yaml" ]]; then
|
||||
echo "docker-compose.yaml is missing from the current dir. Please run this command from a docker-compose environment."
|
||||
exit 1
|
||||
fi
|
||||
if (( $# != 2 )); then
|
||||
cat << EOF
|
||||
Single test executor
|
||||
|
||||
Usage:
|
||||
|
||||
../test-single.sh <container> <robot_test>
|
||||
|
||||
container: Name of the running docker-compose container (docker-compose.yaml is required in the current directory)
|
||||
|
||||
robot_test: name of the robot test or directory relative to the smoketest dir.
|
||||
|
||||
|
||||
|
||||
EOF
|
||||
|
||||
fi
|
||||
|
||||
# shellcheck source=testlib.sh
|
||||
source "$COMPOSE_DIR/../testlib.sh"
|
||||
|
||||
execute_robot_test "$1" "$2"
|
||||
|
||||
generate_report
|
98
hadoop-ozone/dist/src/main/compose/testlib.sh
vendored
Executable file
98
hadoop-ozone/dist/src/main/compose/testlib.sh
vendored
Executable file
@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
set -e
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
|
||||
|
||||
COMPOSE_ENV_NAME=$(basename "$COMPOSE_DIR")
|
||||
COMPOSE_FILE=$COMPOSE_DIR/docker-compose.yaml
|
||||
RESULT_DIR="$COMPOSE_DIR/result"
|
||||
RESULT_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/compose/$(basename "$COMPOSE_ENV_NAME")/result"
|
||||
SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest"
|
||||
|
||||
#delete previous results
|
||||
rm -rf "$RESULT_DIR"
|
||||
mkdir -p "$RESULT_DIR"
|
||||
#Should be writeable from the docker containers where user is different.
|
||||
chmod ogu+w "$RESULT_DIR"
|
||||
|
||||
## @description wait until 3 datanodes are up (or 30 seconds)
|
||||
## @param the docker-compose file
|
||||
wait_for_datanodes(){
|
||||
|
||||
#Reset the timer
|
||||
SECONDS=0
|
||||
|
||||
#Don't give it up until 30 seconds
|
||||
while [[ $SECONDS -lt 90 ]]; do
|
||||
|
||||
#This line checks the number of HEALTHY datanodes registered in scm over the
|
||||
# jmx HTTP servlet
|
||||
datanodes=$(docker-compose -f "$1" exec -T scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value')
|
||||
if [[ "$datanodes" == "3" ]]; then
|
||||
|
||||
#It's up and running. Let's return from the function.
|
||||
echo "$datanodes datanodes are up and registered to the scm"
|
||||
return
|
||||
else
|
||||
|
||||
#Print it only if a number. Could be not a number if scm is not yet started
|
||||
if [[ "$datanodes" ]]; then
|
||||
echo "$datanodes datanode is up and healthy (until now)"
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
echo "WARNING! Datanodes are not started successfully. Please check the docker-compose files"
|
||||
}
|
||||
|
||||
## @description Starts a docker-compose based test environment
|
||||
start_docker_env(){
|
||||
docker-compose -f "$COMPOSE_FILE" down
|
||||
docker-compose -f "$COMPOSE_FILE" up -d --scale datanode=3
|
||||
wait_for_datanodes "$COMPOSE_FILE"
|
||||
sleep 10
|
||||
}
|
||||
|
||||
## @description Execute robot tests in a specific container.
|
||||
## @param Name of the container in the docker-compose file
|
||||
## @param robot test file or directory relative to the smoketest dir
|
||||
execute_robot_test(){
|
||||
CONTAINER="$1"
|
||||
TEST="$2"
|
||||
TEST_NAME=$(basename "$TEST")
|
||||
TEST_NAME=${TEST_NAME%.*}
|
||||
set +e
|
||||
OUTPUT_NAME="$COMPOSE_ENV_NAME-$TEST_NAME-$CONTAINER"
|
||||
docker-compose -f "$COMPOSE_FILE" exec -e SECURITY_ENABLED="${SECURITY_ENABLED}" -T "$CONTAINER" python -m robot --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$RESULT_DIR_INSIDE/robot-$OUTPUT_NAME.xml" "$SMOKETEST_DIR_INSIDE/$TEST"
|
||||
set -e
|
||||
|
||||
}
|
||||
|
||||
## @description Stops a docker-compose based test environment (with saving the logs)
|
||||
stop_docker_env(){
|
||||
docker-compose -f "$COMPOSE_FILE" logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log"
|
||||
if [ "${KEEP_RUNNING:-false}" = false ]; then
|
||||
docker-compose -f "$COMPOSE_FILE" down
|
||||
fi
|
||||
}
|
||||
|
||||
## @description Generate robot framework reports based on the saved results.
|
||||
generate_report(){
|
||||
#Generate the combined output and return with the right exit code (note: robot = execute test, rebot = generate output)
|
||||
docker run --rm -v "$DIR/..:${OZONE_DIR:-/opt/hadoop}" apache/hadoop-runner rebot -d "$RESULT_DIR_INSIDE" "$RESULT_DIR_INSIDE/robot-*.xml"
|
||||
}
|
35
hadoop-ozone/dist/src/main/smoketest/README.md
vendored
35
hadoop-ozone/dist/src/main/smoketest/README.md
vendored
@ -27,4 +27,37 @@ The argument of the `robot` could be any robot file or directory.
|
||||
|
||||
The current configuration in the robot files (hostnames, ports) are adjusted for the docker-based setup but you can easily modify it for any environment.
|
||||
|
||||
The `./test.sh` in this directory can start multiple type of clusters (ozone standalone or ozone + hdfs) and execute the test framework with all of the clusters.
|
||||
# Run tests in docker environment
|
||||
|
||||
In the ./compose folder there are additional test scripts to make it easy to run all tests or run a specific test in a docker environment.
|
||||
|
||||
## Test one environment
|
||||
|
||||
Go to the compose directory and execute the test.sh directly from there:
|
||||
|
||||
```
|
||||
cd compose/ozone
|
||||
./test.sh
|
||||
```
|
||||
|
||||
The results will be saved to the `compose/ozone/results`
|
||||
|
||||
## Run all the tests
|
||||
|
||||
```
|
||||
cd compose
|
||||
./test-all.sh
|
||||
```
|
||||
|
||||
The results will be combined to the `compose/results` folder.
|
||||
|
||||
## Run one specific test case
|
||||
|
||||
Start the compose environment and execute test:
|
||||
|
||||
```
|
||||
cd compose/ozone
|
||||
docker-compose up -d
|
||||
#wait....
|
||||
../test-single.sh scm basic/basic.robot
|
||||
```
|
25
hadoop-ozone/dist/src/main/smoketest/kinit.robot
vendored
Normal file
25
hadoop-ozone/dist/src/main/smoketest/kinit.robot
vendored
Normal file
@ -0,0 +1,25 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
*** Settings ***
|
||||
Documentation Kinit test user
|
||||
Library OperatingSystem
|
||||
Resource commonlib.robot
|
||||
Test Timeout 2 minute
|
||||
|
||||
|
||||
*** Test Cases ***
|
||||
Kinit
|
||||
Kinit test user
|
32
hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
vendored
Normal file
32
hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
*** Settings ***
|
||||
Documentation Test ozone fs with hadoopfs
|
||||
Library OperatingSystem
|
||||
Library String
|
||||
Resource ../commonlib.robot
|
||||
|
||||
*** Variables ***
|
||||
${DATANODE_HOST} datanode
|
||||
${PREFIX} ozone
|
||||
|
||||
*** Test cases ***
|
||||
|
||||
Test hadoop dfs
|
||||
${random} = Generate Random String 5 [NUMBERS]
|
||||
${result} = Execute hdfs dfs -put /opt/hadoop/NOTICE.txt o3fs://bucket1.vol1/${PREFIX}-${random}
|
||||
${result} = Execute hdfs dfs -ls o3fs://bucket1.vol1/
|
||||
Should contain ${PREFIX}-${random}
|
148
hadoop-ozone/dist/src/main/smoketest/test.sh
vendored
148
hadoop-ozone/dist/src/main/smoketest/test.sh
vendored
@ -14,154 +14,14 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
set -e
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
|
||||
RESULT_DIR=result
|
||||
#delete previous results
|
||||
rm -rf "${DIR:?}/$RESULT_DIR"
|
||||
mkdir -p "$DIR/$RESULT_DIR"
|
||||
#Should be writeable from the docker containers where user is different.
|
||||
chmod ogu+w "$DIR/$RESULT_DIR"
|
||||
|
||||
## @description wait until 3 datanodes are up (or 30 seconds)
|
||||
## @param the docker-compose file
|
||||
wait_for_datanodes(){
|
||||
REPLACEMENT="$DIR/../compose/test-all.sh"
|
||||
echo "THIS SCRIPT IS DEPRECATED. Please use $REPLACEMENT instead."
|
||||
|
||||
#Reset the timer
|
||||
SECONDS=0
|
||||
${REPLACEMENT}
|
||||
|
||||
#Don't give it up until 30 seconds
|
||||
while [[ $SECONDS -lt 30 ]]; do
|
||||
|
||||
#This line checks the number of HEALTHY datanodes registered in scm over the
|
||||
# jmx HTTP servlet
|
||||
datanodes=$(docker-compose -f "$1" exec -T scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value')
|
||||
if [[ "$datanodes" == "3" ]]; then
|
||||
|
||||
#It's up and running. Let's return from the function.
|
||||
echo "$datanodes datanodes are up and registered to the scm"
|
||||
return
|
||||
else
|
||||
|
||||
#Print it only if a number. Could be not a number if scm is not yet started
|
||||
if [[ "$datanodes" ]]; then
|
||||
echo "$datanodes datanode is up and healthy (until now)"
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
echo "WARNING! Datanodes are not started successfully. Please check the docker-compose files"
|
||||
}
|
||||
|
||||
## @description Execute selected test suites in a specified docker-compose engironment
|
||||
## @param the name of the docker-compose env relative to ../compose
|
||||
## @param the name of the tests (array of subdir names of the dir of this script)
|
||||
execute_tests(){
|
||||
COMPOSE_DIR=$1
|
||||
COMPOSE_FILE=$DIR/../compose/$COMPOSE_DIR/docker-compose.yaml
|
||||
TESTS=$2
|
||||
echo "-------------------------------------------------"
|
||||
echo "Executing test(s): [${TESTS[*]}]"
|
||||
echo ""
|
||||
echo " Cluster type: $COMPOSE_DIR"
|
||||
echo " Compose file: $COMPOSE_FILE"
|
||||
echo " Output dir: $DIR/$RESULT_DIR"
|
||||
echo " Command to rerun: ./test.sh --keep --env $COMPOSE_DIR $TESTS"
|
||||
echo "-------------------------------------------------"
|
||||
if [ ${COMPOSE_DIR} == "ozonesecure" ]; then
|
||||
SECURITY_ENABLED="true"
|
||||
else
|
||||
SECURITY_ENABLED="false"
|
||||
fi
|
||||
|
||||
docker-compose -f "$COMPOSE_FILE" down
|
||||
docker-compose -f "$COMPOSE_FILE" up -d --scale datanode=3
|
||||
wait_for_datanodes "$COMPOSE_FILE"
|
||||
#TODO: we need to wait for the OM here
|
||||
sleep 10
|
||||
for TEST in "${TESTS[@]}"; do
|
||||
TITLE="Ozone $TEST tests with $COMPOSE_DIR cluster"
|
||||
set +e
|
||||
OUTPUT_NAME="$COMPOSE_DIR-${TEST//\//_}"
|
||||
docker-compose -f "$COMPOSE_FILE" exec -e SECURITY_ENABLED="${SECURITY_ENABLED}" -T om python -m robot --log NONE --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "smoketest/$RESULT_DIR/robot-$OUTPUT_NAME.xml" --logtitle "$TITLE" --reporttitle "$TITLE" "smoketest/$TEST"
|
||||
set -e
|
||||
docker-compose -f "$COMPOSE_FILE" logs > "$DIR/$RESULT_DIR/docker-$OUTPUT_NAME.log"
|
||||
done
|
||||
if [ "$KEEP_RUNNING" = false ]; then
|
||||
docker-compose -f "$COMPOSE_FILE" down
|
||||
fi
|
||||
}
|
||||
RUN_ALL=true
|
||||
KEEP_RUNNING=false
|
||||
POSITIONAL=()
|
||||
while [[ $# -gt 0 ]]
|
||||
do
|
||||
key="$1"
|
||||
|
||||
case $key in
|
||||
--env)
|
||||
DOCKERENV="$2"
|
||||
RUN_ALL=false
|
||||
shift # past argument
|
||||
shift # past value
|
||||
;;
|
||||
--keep)
|
||||
KEEP_RUNNING=true
|
||||
shift # past argument
|
||||
;;
|
||||
--help|-h|-help)
|
||||
cat << EOF
|
||||
|
||||
Acceptance test executor for ozone.
|
||||
|
||||
This is a lightweight test executor for ozone.
|
||||
|
||||
You can run it with
|
||||
|
||||
./test.sh
|
||||
|
||||
Which executes all the tests in all the available environments.
|
||||
|
||||
Or you can run manually one test with
|
||||
|
||||
./test.sh --keep --env ozone-hdfs basic
|
||||
|
||||
--keep means that docker cluster won't be stopped after the test (optional)
|
||||
--env defines the subdirectory under the compose dir
|
||||
The remaining parameters define the test suites under smoketest dir.
|
||||
Could be any directory or robot file relative to the smoketest dir.
|
||||
EOF
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
POSITIONAL+=("$1") # save it in an array for later
|
||||
shift # past argument
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$RUN_ALL" = true ]; then
|
||||
#
|
||||
# This is the definition of the ozone acceptance test suite
|
||||
#
|
||||
# We select the test suites and execute them on multiple type of clusters
|
||||
#
|
||||
TESTS=("basic")
|
||||
execute_tests ozone "${TESTS[@]}"
|
||||
TESTS=("auditparser")
|
||||
execute_tests ozone "${TESTS[@]}"
|
||||
TESTS=("ozonefs")
|
||||
execute_tests ozonefs "${TESTS[@]}"
|
||||
TESTS=("basic")
|
||||
execute_tests ozone-hdfs "${TESTS[@]}"
|
||||
TESTS=("s3")
|
||||
execute_tests ozones3 "${TESTS[@]}"
|
||||
TESTS=("security")
|
||||
execute_tests ozonesecure .
|
||||
else
|
||||
execute_tests "$DOCKERENV" "${POSITIONAL[@]}"
|
||||
fi
|
||||
|
||||
#Generate the combined output and return with the right exit code (note: robot = execute test, rebot = generate output)
|
||||
docker run --rm -v "$DIR/..:/opt/hadoop" apache/hadoop-runner rebot -d "smoketest/$RESULT_DIR" "smoketest/$RESULT_DIR/robot-*.xml"
|
||||
cp -r "$DIR/../compose/result" "$DIR"
|
||||
|
Loading…
Reference in New Issue
Block a user