diff --git a/hadoop-ozone/.gitignore b/hadoop-ozone/.gitignore new file mode 100644 index 0000000000..93c683135f --- /dev/null +++ b/hadoop-ozone/.gitignore @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*~ +*.pyc +.blockade +.cache +__pycache__ diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index 6178dfe72a..c9f81bd997 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -83,6 +83,7 @@ run mkdir -p ./bin run mkdir -p ./sbin run mkdir -p ./etc run mkdir -p ./libexec +run mkdir -p ./tests run cp -r "${ROOT}/hadoop-common-project/hadoop-common/src/main/conf" "etc/hadoop" run cp "${ROOT}/hadoop-ozone/dist/src/main/conf/om-audit-log4j2.properties" "etc/hadoop" @@ -108,6 +109,9 @@ run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/workers.sh" "sb run cp "${ROOT}/hadoop-ozone/common/src/main/bin/start-ozone.sh" "sbin/" run cp "${ROOT}/hadoop-ozone/common/src/main/bin/stop-ozone.sh" "sbin/" +# fault injection tests +run cp -r "${ROOT}/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade" tests + #shaded datanode service run mkdir -p "./share/hadoop/ozoneplugin" run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar" @@ -119,7 +123,6 @@ cp -r "${ROOT}/hadoop-hdds/docs/target/classes/docs" ./ #compose files are preprocessed: properties (eg. project.version) are replaced first by maven. run cp -p -R "${ROOT}/hadoop-ozone/dist/target/compose" . run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" . -run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/blockade" . run cp -p -r "${ROOT}/hadoop-ozone/dist/target/k8s" kubernetes run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/Dockerfile" . diff --git a/hadoop-ozone/fault-injection-test/network-tests/pom.xml b/hadoop-ozone/fault-injection-test/network-tests/pom.xml new file mode 100644 index 0000000000..3b29480c00 --- /dev/null +++ b/hadoop-ozone/fault-injection-test/network-tests/pom.xml @@ -0,0 +1,103 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-ozone-fault-injection-test + 0.5.0-SNAPSHOT + + hadoop-ozone-network-tests + Apache Hadoop Ozone Network Tests + Apache Hadoop Ozone Network Tests + jar + + + + + maven-resources-plugin + 3.1.0 + + + copy-resources + process-resources + + copy-resources + + + ${project.build.directory} + + + src/test/compose + true + + docker-compose.yaml + docker-config + + + + + + + + + + + + + it + + ${basedir}../../dist/target/ozone-${project.version} + + + + + org.codehaus.mojo + exec-maven-plugin + + + integration-test + + exec + + + python + + -m + pytest + -s + ${basedir}/src/test/blockade/ + + + + ${ozone.home} + + + ${project.build.directory} + + + + + + + + + + + + diff --git a/hadoop-ozone/dist/src/main/blockade/README.md b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/README.md similarity index 71% rename from hadoop-ozone/dist/src/main/blockade/README.md rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/README.md index fb58205499..b9f3c73452 100644 --- a/hadoop-ozone/dist/src/main/blockade/README.md +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/README.md @@ -18,19 +18,25 @@ Following python packages need to be installed before running the tests : 1. blockade 2. pytest==2.8.7 +Running test as part of the maven build: + +mvn clean verify -Pit + +Running test as part of the released binary: + You can execute all blockade tests with following command-lines: ``` cd $DIRECTORY_OF_OZONE -python -m pytest -s blockade/ +python -m pytest -s tests/blockade/ ``` You can also execute fewer blockade tests with following command-lines: ``` cd $DIRECTORY_OF_OZONE -python -m pytest -s blockade/ -e.g: python -m pytest -s blockade/test_blockade_datanode_isolation.py +python -m pytest -s tests/blockade/ +e.g: python -m pytest -s tests/blockade/test_blockade_datanode_isolation.py ``` You can change the default 'sleep' interval in the tests with following @@ -38,9 +44,9 @@ command-lines: ``` cd $DIRECTORY_OF_OZONE -python -m pytest -s blockade/ --containerStatusSleep= +python -m pytest -s tests/blockade/ --containerStatusSleep= -e.g: python -m pytest -s blockade/ --containerStatusSleep=720 +e.g: python -m pytest -s tests/blockade/ --containerStatusSleep=720 ``` By default, second phase of the tests will not be run. @@ -49,6 +55,6 @@ command-lines: ``` cd $DIRECTORY_OF_OZONE -python -m pytest -s blockade/ --runSecondPhase=true +python -m pytest -s tests/blockade/ --runSecondPhase=true -``` \ No newline at end of file +``` diff --git a/hadoop-ozone/dist/src/main/blockade/blockadeUtils/__init__.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/blockadeUtils/__init__.py similarity index 100% rename from hadoop-ozone/dist/src/main/blockade/blockadeUtils/__init__.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/blockadeUtils/__init__.py diff --git a/hadoop-ozone/dist/src/main/blockade/blockadeUtils/blockade.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/blockadeUtils/blockade.py similarity index 100% rename from hadoop-ozone/dist/src/main/blockade/blockadeUtils/blockade.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/blockadeUtils/blockade.py diff --git a/hadoop-ozone/dist/src/main/blockade/clusterUtils/__init__.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/__init__.py similarity index 100% rename from hadoop-ozone/dist/src/main/blockade/clusterUtils/__init__.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/__init__.py diff --git a/hadoop-ozone/dist/src/main/blockade/clusterUtils/cluster_utils.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/cluster_utils.py similarity index 99% rename from hadoop-ozone/dist/src/main/blockade/clusterUtils/cluster_utils.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/cluster_utils.py index cf67380255..53e3fa037f 100644 --- a/hadoop-ozone/dist/src/main/blockade/clusterUtils/cluster_utils.py +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/clusterUtils/cluster_utils.py @@ -332,4 +332,4 @@ class ClusterUtils(object): datanodes = sorted( list(filter(lambda x: 'datanode' in x, container_list))) client = filter(lambda x: 'ozone_client' in x, container_list) - return om, scm, client, datanodes \ No newline at end of file + return om, scm, client, datanodes diff --git a/hadoop-ozone/dist/src/main/blockade/conftest.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/conftest.py similarity index 100% rename from hadoop-ozone/dist/src/main/blockade/conftest.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/conftest.py diff --git a/hadoop-ozone/dist/src/main/blockade/ozone/__init__.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/__init__.py similarity index 100% rename from hadoop-ozone/dist/src/main/blockade/ozone/__init__.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/__init__.py diff --git a/hadoop-ozone/dist/src/main/blockade/ozone/cluster.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py similarity index 92% rename from hadoop-ozone/dist/src/main/blockade/ozone/cluster.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py index 4347f86c0d..f75b3d2c8c 100644 --- a/hadoop-ozone/dist/src/main/blockade/ozone/cluster.py +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py @@ -21,6 +21,7 @@ import re import subprocess import yaml import util +from os import environ from subprocess import call from blockadeUtils.blockade import Blockade @@ -44,11 +45,18 @@ class Configuration: """ def __init__(self): - __parent_dir__ = os.path.dirname(os.path.dirname( - os.path.dirname(os.path.realpath(__file__)))) - self.docker_compose_file = os.path.join(__parent_dir__, - "compose", "ozoneblockade", - "docker-compose.yaml") + if "MAVEN_TEST" in os.environ: + compose_dir = environ.get("MAVEN_TEST") + self.docker_compose_file = os.path.join(compose_dir, "docker-compose.yaml") + elif "OZONE_HOME" in os.environ: + compose_dir = os.path.join(environ.get("OZONE_HOME"), "compose", "ozoneblockade") + self.docker_compose_file = os.path.join(compose_dir, "docker-compose.yaml") + else: + __parent_dir__ = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.realpath(__file__))))) + self.docker_compose_file = os.path.join(__parent_dir__, + "compose", "ozoneblockade", + "docker-compose.yaml") self._datanode_count = 3 os.environ["DOCKER_COMPOSE_FILE"] = self.docker_compose_file @@ -292,4 +300,4 @@ class Cluster(object): container_states_dn = self.get_container_states(datanode) if container_states_dn and container_states_dn.popitem()[1] == state: return True - return False \ No newline at end of file + return False diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_client_failure.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py similarity index 88% rename from hadoop-ozone/dist/src/main/blockade/test_blockade_client_failure.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py index 8c0b518493..9e1b04f682 100644 --- a/hadoop-ozone/dist/src/main/blockade/test_blockade_client_failure.py +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py @@ -19,14 +19,24 @@ import os import re import time import logging +from os import environ from blockadeUtils.blockade import Blockade from clusterUtils.cluster_utils import ClusterUtils logger = logging.getLogger(__name__) -parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -FILE = os.path.join(parent_dir, "compose", "ozoneblockade", - "docker-compose.yaml") +if "MAVEN_TEST" in os.environ: + compose_dir = environ.get("MAVEN_TEST") + FILE = os.path.join(compose_dir, "docker-compose.yaml") +elif "OZONE_HOME" in os.environ: + compose_dir = environ.get("OZONE_HOME") + FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") +else: + parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") + os.environ["DOCKER_COMPOSE_FILE"] = FILE SCALE = 3 CONTAINER_LIST = [] @@ -118,4 +128,4 @@ def test_client_failure_isolate_one_datanode(): test_key_name, "/tmp/") key_checksum = ClusterUtils.find_checksum(FILE, "/tmp/%s" % test_key_name) - assert key_checksum == ORIG_CHECKSUM \ No newline at end of file + assert key_checksum == ORIG_CHECKSUM diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_datanode_isolation.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py similarity index 99% rename from hadoop-ozone/dist/src/main/blockade/test_blockade_datanode_isolation.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py index dfa1b703ba..85d99e213d 100644 --- a/hadoop-ozone/dist/src/main/blockade/test_blockade_datanode_isolation.py +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py @@ -22,7 +22,6 @@ from ozone.cluster import Cluster logger = logging.getLogger(__name__) - def setup_function(function): global cluster cluster = Cluster.create() @@ -135,4 +134,4 @@ def test_datanode_isolation_all(): util.wait_until( lambda: cluster.container_state_predicate_all_closed(cluster.datanodes), int(os.environ["CONTAINER_STATUS_SLEEP"]), 10) - assert cluster.container_state_predicate_all_closed(cluster.datanodes) \ No newline at end of file + assert cluster.container_state_predicate_all_closed(cluster.datanodes) diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_flaky.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_flaky.py similarity index 79% rename from hadoop-ozone/dist/src/main/blockade/test_blockade_flaky.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_flaky.py index a79bd4fcc2..6f1df18bec 100644 --- a/hadoop-ozone/dist/src/main/blockade/test_blockade_flaky.py +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_flaky.py @@ -19,14 +19,24 @@ import os import logging import random import pytest +from os import environ from blockadeUtils.blockade import Blockade from ozone.cluster import Cluster logger = logging.getLogger(__name__) -parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -FILE = os.path.join(parent_dir, "compose", "ozoneblockade", - "docker-compose.yaml") +if "MAVEN_TEST" in os.environ: + compose_dir = environ.get("MAVEN_TEST") + FILE = os.path.join(compose_dir, "docker-compose.yaml") +elif "OZONE_HOME" in os.environ: + compose_dir = environ.get("OZONE_HOME") + FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") +else: + parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") + os.environ["DOCKER_COMPOSE_FILE"] = FILE SCALE = 6 CONTAINER_LIST = [] @@ -64,4 +74,4 @@ def test_flaky(flaky_node): Blockade.make_flaky(flaky_container_name) Blockade.blockade_status() exit_code, output = cluster.run_freon(1, 1, 1, 10240) - assert exit_code == 0, "freon run failed with output=[%s]" % output \ No newline at end of file + assert exit_code == 0, "freon run failed with output=[%s]" % output diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py similarity index 91% rename from hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py index 8493ce0272..86d5311d19 100644 --- a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure.py +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py @@ -19,13 +19,23 @@ import os import time import logging import re +from os import environ from blockadeUtils.blockade import Blockade from clusterUtils.cluster_utils import ClusterUtils logger = logging.getLogger(__name__) -parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -FILE = os.path.join(parent_dir, "compose", "ozoneblockade", - "docker-compose.yaml") +if "MAVEN_TEST" in os.environ: + compose_dir = environ.get("MAVEN_TEST") + FILE = os.path.join(compose_dir, "docker-compose.yaml") +elif "OZONE_HOME" in os.environ: + compose_dir = environ.get("OZONE_HOME") + FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") +else: + parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") + os.environ["DOCKER_COMPOSE_FILE"] = FILE SCALE = 3 INCREASED_SCALE = 5 @@ -146,4 +156,4 @@ def test_one_dn_isolate_other_dn(run_second_phase): "The container should have at least three closed replicas." _, output = \ ClusterUtils.run_freon(FILE, 1, 1, 1, 10240, "RATIS", "THREE") - assert re.search("Status: Success", output) is not None \ No newline at end of file + assert re.search("Status: Success", output) is not None diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_three_nodes_isolate.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py similarity index 94% rename from hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_three_nodes_isolate.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py index 0e50025847..ab4c2d4869 100644 --- a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_three_nodes_isolate.py +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py @@ -19,13 +19,23 @@ import os import time import logging import re +from os import environ from blockadeUtils.blockade import Blockade from clusterUtils.cluster_utils import ClusterUtils logger = logging.getLogger(__name__) -parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -FILE = os.path.join(parent_dir, "compose", "ozoneblockade", - "docker-compose.yaml") +if "MAVEN_TEST" in os.environ: + compose_dir = environ.get("MAVEN_TEST") + FILE = os.path.join(compose_dir, "docker-compose.yaml") +elif "OZONE_HOME" in os.environ: + compose_dir = environ.get("OZONE_HOME") + FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") +else: + parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") + os.environ["DOCKER_COMPOSE_FILE"] = FILE SCALE = 3 INCREASED_SCALE = 5 @@ -222,4 +232,4 @@ def test_three_dns_isolate_threescmfailure(run_second_phase): count_closed_container_datanodes = filter( lambda x: x == 'CLOSED', all_datanodes_container_status) assert len(count_closed_container_datanodes) == 3, \ - "The container should have three closed replicas." \ No newline at end of file + "The container should have three closed replicas." diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_two_nodes.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py similarity index 92% rename from hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_two_nodes.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py index b8df2fa54b..03da7d03ed 100644 --- a/hadoop-ozone/dist/src/main/blockade/test_blockade_mixed_failure_two_nodes.py +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py @@ -19,13 +19,23 @@ import os import time import logging import re +from os import environ from blockadeUtils.blockade import Blockade from clusterUtils.cluster_utils import ClusterUtils logger = logging.getLogger(__name__) -parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -FILE = os.path.join(parent_dir, "compose", "ozoneblockade", - "docker-compose.yaml") +if "MAVEN_TEST" in os.environ: + compose_dir = environ.get("MAVEN_TEST") + FILE = os.path.join(compose_dir, "docker-compose.yaml") +elif "OZONE_HOME" in os.environ: + compose_dir = environ.get("OZONE_HOME") + FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") +else: + parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") + os.environ["DOCKER_COMPOSE_FILE"] = FILE SCALE = 3 INCREASED_SCALE = 5 @@ -170,4 +180,4 @@ def test_two_dns_isolate_scm_different_partition(run_second_phase): assert len(count_closed_container_datanodes) >= 3 _, output = \ ClusterUtils.run_freon(FILE, 1, 1, 1, 10240, "RATIS", "THREE") - assert re.search("Status: Success", output) is not None \ No newline at end of file + assert re.search("Status: Success", output) is not None diff --git a/hadoop-ozone/dist/src/main/blockade/test_blockade_scm_isolation.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py similarity index 91% rename from hadoop-ozone/dist/src/main/blockade/test_blockade_scm_isolation.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py index 06f4263746..47bbb76b76 100644 --- a/hadoop-ozone/dist/src/main/blockade/test_blockade_scm_isolation.py +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py @@ -19,13 +19,23 @@ import os import time import re import logging +from os import environ from blockadeUtils.blockade import Blockade from clusterUtils.cluster_utils import ClusterUtils logger = logging.getLogger(__name__) -parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -FILE = os.path.join(parent_dir, "compose", "ozoneblockade", - "docker-compose.yaml") +if "MAVEN_TEST" in os.environ: + compose_dir = environ.get("MAVEN_TEST") + FILE = os.path.join(compose_dir, "docker-compose.yaml") +elif "OZONE_HOME" in os.environ: + compose_dir = environ.get("OZONE_HOME") + FILE = os.path.join(compose_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") +else: + parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + FILE = os.path.join(parent_dir, "compose", "ozoneblockade", \ + "docker-compose.yaml") + os.environ["DOCKER_COMPOSE_FILE"] = FILE SCALE = 3 INCREASED_SCALE = 5 @@ -154,4 +164,4 @@ def test_scm_isolation_two_node(run_second_phase): assert len(closed_container_datanodes) >= 3 _, output = \ ClusterUtils.run_freon(FILE, 1, 1, 1, 10240, "RATIS", "THREE") - assert re.search("Status: Success", output) is not None \ No newline at end of file + assert re.search("Status: Success", output) is not None diff --git a/hadoop-ozone/dist/src/main/blockade/util.py b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/util.py similarity index 100% rename from hadoop-ozone/dist/src/main/blockade/util.py rename to hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/util.py diff --git a/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-compose.yaml b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-compose.yaml new file mode 100644 index 0000000000..6c8e0fb78b --- /dev/null +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-compose.yaml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + datanode: + image: ${user.name}/ozone:${project.version} + ports: + - 9864 + command: ["/opt/hadoop/bin/ozone","datanode"] + env_file: + - ./docker-config + om: + image: ${user.name}/ozone:${project.version} + ports: + - 9874:9874 + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/ozone","om"] + scm: + image: ${user.name}/ozone:${project.version} + ports: + - 9876:9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["/opt/hadoop/bin/ozone","scm"] + ozone_client: + image: ${user.name}/ozone:${project.version} + ports: + - 9869 + command: ["tail", "-f","/etc/passwd"] + env_file: + - ./docker-config diff --git a/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config new file mode 100644 index 0000000000..1db1a798d3 --- /dev/null +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE-SITE.XML_ozone.om.address=om +OZONE-SITE.XML_ozone.om.http-address=om:9874 +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=True +OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.handler.type=distributed +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.scm.dead.node.interval=5m +OZONE-SITE.XML_ozone.replication=1 +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR + +#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. +#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm + +#LOG4J2.PROPERTIES_* are for Ozone Audit Logging +LOG4J2.PROPERTIES_monitorInterval=30 +LOG4J2.PROPERTIES_filter=read,write +LOG4J2.PROPERTIES_filter.read.type=MarkerFilter +LOG4J2.PROPERTIES_filter.read.marker=READ +LOG4J2.PROPERTIES_filter.read.onMatch=DENY +LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.type=MarkerFilter +LOG4J2.PROPERTIES_filter.write.marker=WRITE +LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_appenders=console, rolling +LOG4J2.PROPERTIES_appender.console.type=Console +LOG4J2.PROPERTIES_appender.console.name=STDOUT +LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.type=RollingFile +LOG4J2.PROPERTIES_appender.rolling.name=RollingFile +LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log +LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz +LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies +LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 +LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB +LOG4J2.PROPERTIES_loggers=audit +LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger +LOG4J2.PROPERTIES_logger.audit.name=OMAudit +LOG4J2.PROPERTIES_logger.audit.level=INFO +LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling +LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile +LOG4J2.PROPERTIES_rootLogger.level=INFO +LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout +LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT diff --git a/hadoop-ozone/fault-injection-test/pom.xml b/hadoop-ozone/fault-injection-test/pom.xml new file mode 100644 index 0000000000..395c5340bf --- /dev/null +++ b/hadoop-ozone/fault-injection-test/pom.xml @@ -0,0 +1,35 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-ozone + 0.5.0-SNAPSHOT + + hadoop-ozone-fault-injection-test + 0.5.0-SNAPSHOT + Apache Hadoop Ozone Fault Injection Tests + Apache Hadoop Ozone Fault Injection Tests + pom + + + network-tests + + + diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index 2d80c3a284..4a6df28534 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -53,6 +53,7 @@ ozone-recon-codegen upgrade csi + fault-injection-test @@ -137,7 +138,6 @@ ${ozone.version} test-jar - org.apache.hadoop hadoop-hdds-common diff --git a/pom.ozone.xml b/pom.ozone.xml index 413ca3bb4e..ff841bd6dd 100644 --- a/pom.ozone.xml +++ b/pom.ozone.xml @@ -18,7 +18,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 4.0.0 org.apache.hadoop hadoop-main-ozone - 0.4.0-SNAPSHOT + 0.5.0-SNAPSHOT Apache Hadoop Ozone Main Apache Hadoop Ozone Main pom