From 62f817d32e6304b4cc1f9b909e7b815316b3a021 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 24 Sep 2018 10:10:11 -0700 Subject: [PATCH] HDDS-447. Separate ozone-dist and hadoop-dist projects with real classpath separation. Contributed by Elek Marton. --- dev-support/bin/ozone-dist-layout-stitching | 181 ----------------- hadoop-dist/pom.xml | 118 ----------- hadoop-hdds/client/pom.xml | 5 - hadoop-hdds/common/pom.xml | 2 - hadoop-hdds/container-service/pom.xml | 5 - hadoop-hdds/framework/pom.xml | 5 - hadoop-hdds/pom.xml | 30 +++ hadoop-hdds/server-scm/pom.xml | 50 ++--- .../{ => resources}/webapps/scm/index.html | 0 .../{ => resources}/webapps/scm/main.html | 0 .../webapps/scm/scm-overview.html | 0 .../main/{ => resources}/webapps/scm/scm.js | 0 hadoop-hdds/tools/pom.xml | 8 - hadoop-ozone/client/pom.xml | 6 - hadoop-ozone/common/pom.xml | 5 - hadoop-ozone/common/src/main/bin/ozone | 50 ++++- .../src/main/shellprofile.d/hadoop-ozone.sh | 24 --- hadoop-ozone/datanode/pom.xml | 49 +++++ .../dev-support/bin/dist-layout-stitching | 113 +++++++++++ .../dist/dev-support/bin/dist-tar-stitching | 2 + hadoop-ozone/dist/pom.xml | 184 ++++++++++++++++++ .../dist}/src/main/compose/README.md | 0 .../dist}/src/main/compose/ozone-hdfs/.env | 0 .../compose/ozone-hdfs/docker-compose.yaml | 0 .../src/main/compose/ozone-hdfs/docker-config | 0 .../dist}/src/main/compose/ozone/.env | 0 .../main/compose/ozone/docker-compose.yaml | 0 .../src/main/compose/ozone/docker-config | 0 .../main/compose/ozonefs/docker-compose.yaml | 0 .../src/main/compose/ozonefs/docker-config | 0 .../dist}/src/main/compose/ozoneperf/.env | 0 .../src/main/compose/ozoneperf/README.md | 0 .../src/main/compose/ozoneperf/compose-all.sh | 0 .../ozoneperf/docker-compose-freon.yaml | 0 .../compose/ozoneperf/docker-compose.yaml | 2 +- .../src/main/compose/ozoneperf/docker-config | 1 - .../dist}/src/main/compose/ozoneperf/init.sh | 0 .../src/main/compose/ozoneperf/prometheus.yml | 0 .../compose/ozonescripts/.ssh/authorized_keys | 0 .../src/main/compose/ozonescripts/.ssh/config | 0 .../compose/ozonescripts/.ssh/environment | 0 .../src/main/compose/ozonescripts/.ssh/id_rsa | 0 .../main/compose/ozonescripts/.ssh/id_rsa.pub | 0 .../src/main/compose/ozonescripts/Dockerfile | 0 .../src/main/compose/ozonescripts/README.md | 0 .../compose/ozonescripts/docker-compose.yaml | 0 .../main/compose/ozonescripts/docker-config | 0 .../dist}/src/main/compose/ozonescripts/ps.sh | 0 .../src/main/compose/ozonescripts/start.sh | 0 .../src/main/compose/ozonescripts/stop.sh | 0 .../dist}/src/main/ozone/README.txt | 0 .../dist}/src/main/smoketest/README.md | 0 .../src/main/smoketest/basic/basic.robot | 0 .../main/smoketest/basic/ozone-shell.robot | 0 .../dist}/src/main/smoketest/commonlib.robot | 0 .../src/main/smoketest/ozonefs/ozonefs.robot | 0 .../dist}/src/main/smoketest/test.sh | 0 hadoop-ozone/docs/pom.xml | 5 - hadoop-ozone/integration-test/pom.xml | 14 -- hadoop-ozone/objectstore-service/pom.xml | 5 - hadoop-ozone/ozone-manager/pom.xml | 50 ++--- .../webapps/ozoneManager/index.html | 0 .../webapps/ozoneManager/main.css | 0 .../webapps/ozoneManager/main.html | 0 .../webapps/ozoneManager/om-metrics.html | 0 .../webapps/ozoneManager/ozoneManager.js | 0 hadoop-ozone/ozonefs/pom.xml | 2 - hadoop-ozone/pom.xml | 25 ++- hadoop-ozone/tools/pom.xml | 32 ++- .../hadoop/ozone/freon/TestDataValidate.java | 0 .../ozone/freon/TestRandomKeyGenerator.java | 0 .../apache/hadoop/ozone/om/TestOmSQLCli.java | 0 .../hadoop/ozone/scm/TestContainerSQLCli.java | 0 hadoop-project/pom.xml | 5 + 74 files changed, 503 insertions(+), 475 deletions(-) delete mode 100755 dev-support/bin/ozone-dist-layout-stitching rename hadoop-hdds/server-scm/src/main/{ => resources}/webapps/scm/index.html (100%) rename hadoop-hdds/server-scm/src/main/{ => resources}/webapps/scm/main.html (100%) rename hadoop-hdds/server-scm/src/main/{ => resources}/webapps/scm/scm-overview.html (100%) rename hadoop-hdds/server-scm/src/main/{ => resources}/webapps/scm/scm.js (100%) create mode 100644 hadoop-ozone/datanode/pom.xml create mode 100755 hadoop-ozone/dist/dev-support/bin/dist-layout-stitching rename dev-support/bin/ozone-dist-tar-stitching => hadoop-ozone/dist/dev-support/bin/dist-tar-stitching (95%) create mode 100644 hadoop-ozone/dist/pom.xml rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/README.md (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozone-hdfs/.env (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozone-hdfs/docker-compose.yaml (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozone-hdfs/docker-config (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozone/.env (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozone/docker-compose.yaml (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozone/docker-config (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonefs/docker-compose.yaml (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonefs/docker-config (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozoneperf/.env (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozoneperf/README.md (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozoneperf/compose-all.sh (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozoneperf/docker-compose-freon.yaml (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozoneperf/docker-compose.yaml (98%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozoneperf/docker-config (99%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozoneperf/init.sh (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozoneperf/prometheus.yml (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/.ssh/authorized_keys (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/.ssh/config (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/.ssh/environment (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/.ssh/id_rsa (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/.ssh/id_rsa.pub (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/Dockerfile (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/README.md (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/docker-compose.yaml (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/docker-config (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/ps.sh (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/start.sh (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/compose/ozonescripts/stop.sh (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/ozone/README.txt (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/smoketest/README.md (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/smoketest/basic/basic.robot (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/smoketest/basic/ozone-shell.robot (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/smoketest/commonlib.robot (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/smoketest/ozonefs/ozonefs.robot (100%) rename {hadoop-dist => hadoop-ozone/dist}/src/main/smoketest/test.sh (100%) rename hadoop-ozone/ozone-manager/src/main/{ => resources}/webapps/ozoneManager/index.html (100%) rename hadoop-ozone/ozone-manager/src/main/{ => resources}/webapps/ozoneManager/main.css (100%) rename hadoop-ozone/ozone-manager/src/main/{ => resources}/webapps/ozoneManager/main.html (100%) rename hadoop-ozone/ozone-manager/src/main/{ => resources}/webapps/ozoneManager/om-metrics.html (100%) rename hadoop-ozone/ozone-manager/src/main/{ => resources}/webapps/ozoneManager/ozoneManager.js (100%) rename hadoop-ozone/{integration-test => tools}/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java (100%) rename hadoop-ozone/{integration-test => tools}/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java (100%) rename hadoop-ozone/{integration-test => tools}/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java (100%) rename hadoop-ozone/{integration-test => tools}/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java (100%) diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching deleted file mode 100755 index 00854b45f2..0000000000 --- a/dev-support/bin/ozone-dist-layout-stitching +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# project.version -VERSION=$1 - -# project.build.directory -BASEDIR=$2 - -#hdds.version -HDDS_VERSION=$3 - -function run() -{ - declare res - - echo "\$ ${*}" - "${@}" - res=$? - if [[ ${res} != 0 ]]; then - echo - echo "Failed!" - echo - exit "${res}" - fi -} - -function findfileindir() -{ - declare file="$1" - declare dir="${2:-./share}" - declare count - - count=$(find "${dir}" -iname "${file}" | wc -l) - - #shellcheck disable=SC2086 - echo ${count} -} - -function copyifnotexists() -{ - declare src="$1" - declare dest="$2" - - declare srcname - declare destdir - - declare child - declare childpath - - if [[ -f "${src}" ]]; then - srcname=${src##*/} - if [[ "${srcname}" != *.jar || - $(findfileindir "${srcname}") -eq "0" ]]; then - destdir=$(dirname "${dest}") - mkdir -p "${destdir}" - cp -p "${src}" "${dest}" - fi - else - for childpath in "${src}"/*; do - child="${childpath##*/}" - if [[ "${child}" == "doc" || - "${child}" == "webapps" ]]; then - mkdir -p "${dest}/${child}" - cp -r "${src}/${child}"/* "${dest}/${child}" - continue; - fi - copyifnotexists "${src}/${child}" "${dest}/${child}" - done - fi -} - -#Copy all contents as is except the lib. -#for libs check for existence in share directory, if not exist then only copy. -function copy() -{ - declare src="$1" - declare dest="$2" - - declare child - declare childpath - - if [[ -d "${src}" ]]; then - for childpath in "${src}"/*; do - child="${childpath##*/}" - - if [[ "${child}" == "share" ]]; then - copyifnotexists "${src}/${child}" "${dest}/${child}" - else - if [[ -d "${src}/${child}" ]]; then - mkdir -p "${dest}/${child}" - cp -pr "${src}/${child}"/* "${dest}/${child}" - else - cp -pr "${src}/${child}" "${dest}/${child}" - fi - fi - done - fi -} - -# shellcheck disable=SC2164 -ROOT=$(cd "${BASEDIR}"/../..;pwd) -echo -echo "Current directory $(pwd)" -echo -run rm -rf "ozone-${HDDS_VERSION}" -run mkdir "ozone-${HDDS_VERSION}" -run cd "ozone-${HDDS_VERSION}" -run cp -p "${ROOT}/LICENSE.txt" . -run cp -p "${ROOT}/NOTICE.txt" . - -# Copy hadoop-common first so that it have always have all dependencies. -# Remaining projects will copy only libraries which are not present already in 'share' directory. -run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" . - - -# HDDS -run copy "${ROOT}/hadoop-hdds/common/target/hadoop-hdds-common-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/framework/target/hadoop-hdds-server-framework-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/server-scm/target/hadoop-hdds-server-scm-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/container-service/target/hadoop-hdds-container-service-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/client/target/hadoop-hdds-client-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/tools/target/hadoop-hdds-tools-${HDDS_VERSION}" . - -# Ozone -run copy "${ROOT}/hadoop-ozone/common/target/hadoop-ozone-common-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" . - -#shaded ozonefs -mkdir -p "./share/hadoop/ozonefs" -cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" - -#shaded datanode service -mkdir -p "./share/hadoop/ozoneplugin" -cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar" - - -# Optional documentation, could be missing -cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/ -cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/ -cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./ - - -rm sbin/*all.sh -rm sbin/*all.cmd - -#remove test and java sources -find . -name "*tests.jar" | xargs rm -find . -name "*sources.jar" | xargs rm -find . -name jdiff -type d | xargs rm -rf - -#add ozone specific readme - -run cp "${ROOT}/hadoop-dist/src/main/ozone/README.txt" README.txt -#Copy docker compose files and robot tests -run cp -p -r "${ROOT}/hadoop-dist/src/main/compose" . -run cp -p -r "${ROOT}/hadoop-dist/src/main/smoketest" . - -mkdir -p ./share/hadoop/mapreduce -mkdir -p ./share/hadoop/yarn -mkdir -p ./share/hadoop/hdfs -echo -echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}" -echo diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index da05015e43..5447b6ab74 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -176,7 +176,6 @@ ${project.version} ${project.build.directory} - ${hdds.version} @@ -224,123 +223,6 @@ - - hdds - - false - - - - org.apache.hadoop - hadoop-ozone-ozone-manager - provided - - - org.apache.hadoop - hadoop-hdds-server-scm - provided - - - org.apache.hadoop - hadoop-hdds-tools - provided - - - org.apache.hadoop - hadoop-hdds-container-service - provided - - - org.apache.hadoop - hadoop-ozone-objectstore-service - provided - - - org.apache.hadoop - hadoop-ozone-tools - provided - - - org.apache.hadoop - hadoop-ozone-docs - provided - - - org.apache.hadoop - hadoop-ozone-filesystem - provided - - - - - - maven-resources-plugin - - - copy-dockerfile - - copy-resources - - prepare-package - - ${project.build.directory} - - - src/main/docker - true - - - - - - - - org.codehaus.mojo - exec-maven-plugin - - - dist-ozone - prepare-package - - exec - - - ${shell-executable} - ${project.build.directory} - - - - ${basedir}/../dev-support/bin/ozone-dist-layout-stitching - - ${project.version} - ${project.build.directory} - ${hdds.version} - - - - - tar-ozone - package - - exec - - - ${shell-executable} - ${project.build.directory} - - - ${basedir}/../dev-support/bin/ozone-dist-tar-stitching - - ${hdds.version} - ${project.build.directory} - - - - - - - - diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml index 6a4cc9deeb..53d72bb7e4 100644 --- a/hadoop-hdds/client/pom.xml +++ b/hadoop-hdds/client/pom.xml @@ -29,11 +29,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop HDDS Client jar - - hdds - true - - org.apache.hadoop diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index d08a5a9e6a..64ebe9bd91 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -30,8 +30,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> 0.3.0-SNAPSHOT - hdds - true 2.11.0 3.4.2 ${hdds.version} diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index 3d4e58185c..54c5fadce9 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop HDDS Container Service jar - - hdds - true - - org.apache.hadoop diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index cb380e2606..511f32131c 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop HDDS Server Framework jar - - hdds - true - - org.apache.hadoop diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml index ce4f234bc2..59dfa79fc6 100644 --- a/hadoop-hdds/pom.xml +++ b/hadoop-hdds/pom.xml @@ -119,6 +119,36 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + org.apache.maven.plugins + maven-dependency-plugin + + + add-classpath-descriptor + package + + build-classpath + + + true + $HDDS_LIB_JARS_DIR + true + runtime + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index 52f42e1c89..f34e848251 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop HDDS SCM Server jar - - hdds - true - - org.apache.hadoop @@ -104,28 +99,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - - org.apache.maven.plugins - maven-antrun-plugin - - - copy web resources - compile - - run - - - - - - - - - - - - - org.apache.maven.plugins maven-dependency-plugin @@ -136,19 +109,20 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> unpack + + + + org.apache.hadoop + hadoop-hdds-server-framework + ${project.build.outputDirectory} + + webapps/static/**/*.* + + + true + - - - - org.apache.hadoop - hadoop-hdds-server-framework - ${project.build.directory}/ - webapps/static/**/*.* - - - true - diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/index.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html similarity index 100% rename from hadoop-hdds/server-scm/src/main/webapps/scm/index.html rename to hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/main.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html similarity index 100% rename from hadoop-hdds/server-scm/src/main/webapps/scm/main.html rename to hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html similarity index 100% rename from hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html rename to hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js similarity index 100% rename from hadoop-hdds/server-scm/src/main/webapps/scm/scm.js rename to hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml index 852c81d052..37c7d9d74e 100644 --- a/hadoop-hdds/tools/pom.xml +++ b/hadoop-hdds/tools/pom.xml @@ -29,26 +29,18 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop HDDS Tools jar - - hdds - true - - org.apache.hadoop hadoop-hdds-common - provided org.apache.hadoop hadoop-hdds-client - provided org.apache.hadoop hadoop-common - provided commons-cli diff --git a/hadoop-ozone/client/pom.xml b/hadoop-ozone/client/pom.xml index bbd075967e..e471710c29 100644 --- a/hadoop-ozone/client/pom.xml +++ b/hadoop-ozone/client/pom.xml @@ -28,16 +28,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop Ozone Client jar - - ozone - true - - org.apache.hadoop hadoop-ozone-common - provided \ No newline at end of file diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index 37a2cb962b..942576b610 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop Ozone Common jar - - ozone - true - - diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index 6bf8b012f2..d173ec6d58 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -68,6 +68,7 @@ function ozonecmd_case datanode) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-datanode" ;; envvars) echo "JAVA_HOME='${JAVA_HOME}'" @@ -86,42 +87,50 @@ function ozonecmd_case ;; freon) HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.Freon + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; genesis) HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; getozoneconf) HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf; + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; om) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager HDFS_OM_OPTS="${HDFS_OM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties" HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_OM_OPTS}" + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager" ;; sh | shell) HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell - ;; - noz) - HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SQLCLI + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager" ;; scm) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManager' hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}" + OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-server-scm" + ;; fs) HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; scmcli) HADOOP_CLASSNAME=org.apache.hadoop.hdds.scm.cli.SCMCLI + OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-tools" ;; version) HADOOP_CLASSNAME=org.apache.hadoop.ozone.util.OzoneVersionInfo + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-common" ;; genconf) HADOOP_CLASSNAME=org.apache.hadoop.ozone.genconf.GenerateOzoneRequiredConfigurations + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; *) HADOOP_CLASSNAME="${subcmd}" @@ -161,6 +170,7 @@ fi HADOOP_SUBCMD=$1 shift + if hadoop_need_reexec ozone "${HADOOP_SUBCMD}"; then hadoop_uservar_su ozone "${HADOOP_SUBCMD}" \ "${MYNAME}" \ @@ -180,6 +190,40 @@ else ozonecmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}" fi + +# +# Setting up classpath based on the generate classpath descriptors +# +if [ ! "$OZONE_RUN_ARTIFACT_NAME" ]; then + echo "ERROR: Ozone components require to set OZONE_RUN_ARTIFACT_NAME to set the classpath" + exit -1 +fi +export HDDS_LIB_JARS_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib" +CLASSPATH_FILE="${HADOOP_HDFS_HOME}/share/ozone/classpath/${OZONE_RUN_ARTIFACT_NAME}.classpath" +if [ ! "$CLASSPATH_FILE" ]; then + echo "ERROR: Classpath file descriptor $CLASSPATH_FILE is missing" + exit -1 +fi +# shellcheck disable=SC1090,SC2086 +source $CLASSPATH_FILE +OIFS=$IFS +IFS=':' +# shellcheck disable=SC2154 +for jar in $classpath; do + hadoop_add_classpath "$jar" +done +hadoop_add_classpath "${HADOOP_HDFS_HOME}/share/ozone/web" + +#We need to add the artifact manually as it's not part the generated classpath desciptor +ARTIFACT_LIB_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib" +MAIN_ARTIFACT=$(find "$ARTIFACT_LIB_DIR" -name "${OZONE_RUN_ARTIFACT_NAME}-*.jar") +if [ ! "$MAIN_ARTIFACT" ]; then + echo "ERROR: Component jar file $MAIN_ARTIFACT is missing from ${HADOOP_HDFS_HOME}/share/ozone/lib" +fi +hadoop_add_classpath "${MAIN_ARTIFACT}" +IFS=$OIFS + + hadoop_add_client_opts if [[ ${HADOOP_WORKER_MODE} = true ]]; then diff --git a/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh b/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh index 99885abe22..3fff7f5f35 100644 --- a/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh +++ b/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh @@ -19,27 +19,3 @@ if [[ "${HADOOP_SHELL_EXECNAME}" = ozone ]]; then fi -## @description Profile for hdds/ozone components. -## @audience private -## @stability evolving -function _ozone_hadoop_classpath -{ - # - # get all of the ozone jars+config in the path - # - - if [[ -d "${HADOOP_HDFS_HOME}/${HDDS_DIR}/webapps" ]]; then - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_DIR}" - fi - - if [[ -d "${HADOOP_HDFS_HOME}/${HDDS_DIR}/webapps" ]]; then - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_DIR}" - fi - - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_LIB_JARS_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_LIB_JARS_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONEFS_DIR}"'/*' - -} diff --git a/hadoop-ozone/datanode/pom.xml b/hadoop-ozone/datanode/pom.xml new file mode 100644 index 0000000000..02995f5348 --- /dev/null +++ b/hadoop-ozone/datanode/pom.xml @@ -0,0 +1,49 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-ozone + 0.3.0-SNAPSHOT + + hadoop-ozone-datanode + Apache Hadoop Ozone Datanode + jar + 0.3.0-SNAPSHOT + + + + org.apache.hadoop + hadoop-common + compile + + + org.apache.hadoop + hadoop-hdfs + compile + + + org.apache.hadoop + hadoop-hdds-container-service + + + org.apache.hadoop + hadoop-ozone-objectstore-service + + + diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching new file mode 100755 index 0000000000..3f102fa01b --- /dev/null +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# project.build.directory +BASEDIR=$1 + +#hdds.version +HDDS_VERSION=$2 + +## @audience private +## @stability evolving +function run() +{ + declare res + + echo "\$ ${*}" + "${@}" + res=$? + if [[ ${res} != 0 ]]; then + echo + echo "Failed!" + echo + exit "${res}" + fi +} + +## @audience private +## @stability evolving +function findfileindir() +{ + declare file="$1" + declare dir="${2:-./share}" + declare count + + count=$(find "${dir}" -iname "${file}" | wc -l) + + #shellcheck disable=SC2086 + echo ${count} +} + + +# shellcheck disable=SC2164 +ROOT=$(cd "${BASEDIR}"/../../..;pwd) +echo +echo "Current directory $(pwd)" +echo + +run rm -rf "ozone-${HDDS_VERSION}" +run mkdir "ozone-${HDDS_VERSION}" +run cd "ozone-${HDDS_VERSION}" +run cp -p "${ROOT}/LICENSE.txt" . +run cp -p "${ROOT}/NOTICE.txt" . +run cp -p "${ROOT}/README.txt" . + +run mkdir -p ./share/hadoop/mapreduce +run mkdir -p ./share/hadoop/ozone +run mkdir -p ./share/hadoop/hdds +run mkdir -p ./share/hadoop/yarn +run mkdir -p ./share/hadoop/hdfs +run mkdir -p ./share/hadoop/common +run mkdir -p ./share/ozone/web +run mkdir -p ./bin +run mkdir -p ./sbin +run mkdir -p ./etc +run mkdir -p ./libexec + +run cp -r "${ROOT}/hadoop-common-project/hadoop-common/src/main/conf" "etc/hadoop" + +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop" "bin/" +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd" "bin/" +run cp "${ROOT}/hadoop-ozone/common/src/main/bin/ozone" "bin/" + +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh" "libexec/" +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd" "libexec/" +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh" "libexec/" +run cp "${ROOT}/hadoop-ozone/common/src/main/bin/ozone-config.sh" "libexec/" +run cp -r "${ROOT}/hadoop-ozone/common/src/main/shellprofile.d" "libexec/" + + +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh" "sbin/" +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/workers.sh" "sbin/" +run cp "${ROOT}/hadoop-ozone/common/src/main/bin/start-ozone.sh" "sbin/" +run cp "${ROOT}/hadoop-ozone/common/src/main/bin/stop-ozone.sh" "sbin/" + +#shaded ozonefs +run mkdir -p "./share/hadoop/ozonefs" +run cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" + +#shaded datanode service +run mkdir -p "./share/hadoop/ozoneplugin" +run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar" + + +# Optional documentation, could be missing +cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/ +cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/ + +#Copy docker compose files +run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/compose" . +run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" . diff --git a/dev-support/bin/ozone-dist-tar-stitching b/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching similarity index 95% rename from dev-support/bin/ozone-dist-tar-stitching rename to hadoop-ozone/dist/dev-support/bin/dist-tar-stitching index 93d0525e7e..c94e7d0eb6 100755 --- a/dev-support/bin/ozone-dist-tar-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching @@ -21,6 +21,8 @@ VERSION=$1 # project.build.directory BASEDIR=$2 +## @audience private +## @stability evolving function run() { declare res diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml new file mode 100644 index 0000000000..ba9f14fe91 --- /dev/null +++ b/hadoop-ozone/dist/pom.xml @@ -0,0 +1,184 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-ozone + 0.3.0-SNAPSHOT + + hadoop-ozone-dist + Apache Hadoop Ozone Distribution + pom + 0.3.0-SNAPSHOT + + UTF-8 + true + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-classpath-files + package + + copy + + + target/ozone-${ozone.version}/share/ozone/classpath + + + + org.apache.hadoop + hadoop-hdds-server-scm + ${hdds.version} + classpath + hadoop-hdds-server-scm.classpath + + + org.apache.hadoop + hadoop-hdds-tools + ${hdds.version} + classpath + hadoop-hdds-tools.classpath + + + org.apache.hadoop + hadoop-ozone-ozone-manager + ${ozone.version} + classpath + hadoop-ozone-ozone-manager.classpath + + + + org.apache.hadoop + hadoop-ozone-tools + ${ozone.version} + classpath + hadoop-ozone-tools.classpath + + + org.apache.hadoop + hadoop-ozone-common + ${ozone.version} + classpath + hadoop-ozone-common.classpath + + + org.apache.hadoop + hadoop-ozone-datanode + ${ozone.version} + classpath + hadoop-ozone-datanode.classpath + + + + + + copy-jars + package + + copy-dependencies + + + target/ozone-${ozone.version}/share/ozone/lib + + runtime + + + + + + org.codehaus.mojo + exec-maven-plugin + + + dist + prepare-package + + exec + + + ${shell-executable} + ${project.build.directory} + + + ${basedir}/dev-support/bin/dist-layout-stitching + + ${project.build.directory} + ${hdds.version} + + + + + tar-ozone + package + + exec + + + ${shell-executable} + ${project.build.directory} + + + ${basedir}/dev-support/bin/dist-tar-stitching + + ${hdds.version} + ${project.build.directory} + + + + + + + + + + + org.apache.hadoop + hadoop-hdds-tools + + + org.apache.hadoop + hadoop-hdds-server-scm + + + org.apache.hadoop + hadoop-hdds-container-service + + + org.apache.hadoop + hadoop-ozone-ozone-manager + + + org.apache.hadoop + hadoop-ozone-tools + + + org.apache.hadoop + hadoop-ozone-common + + + org.apache.hadoop + hadoop-ozone-datanode + + + diff --git a/hadoop-dist/src/main/compose/README.md b/hadoop-ozone/dist/src/main/compose/README.md similarity index 100% rename from hadoop-dist/src/main/compose/README.md rename to hadoop-ozone/dist/src/main/compose/README.md diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/.env b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env similarity index 100% rename from hadoop-dist/src/main/compose/ozone-hdfs/.env rename to hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml similarity index 100% rename from hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml rename to hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config similarity index 100% rename from hadoop-dist/src/main/compose/ozone-hdfs/docker-config rename to hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config diff --git a/hadoop-dist/src/main/compose/ozone/.env b/hadoop-ozone/dist/src/main/compose/ozone/.env similarity index 100% rename from hadoop-dist/src/main/compose/ozone/.env rename to hadoop-ozone/dist/src/main/compose/ozone/.env diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml similarity index 100% rename from hadoop-dist/src/main/compose/ozone/docker-compose.yaml rename to hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config similarity index 100% rename from hadoop-dist/src/main/compose/ozone/docker-config rename to hadoop-ozone/dist/src/main/compose/ozone/docker-config diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml similarity index 100% rename from hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml rename to hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-config b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config similarity index 100% rename from hadoop-dist/src/main/compose/ozonefs/docker-config rename to hadoop-ozone/dist/src/main/compose/ozonefs/docker-config diff --git a/hadoop-dist/src/main/compose/ozoneperf/.env b/hadoop-ozone/dist/src/main/compose/ozoneperf/.env similarity index 100% rename from hadoop-dist/src/main/compose/ozoneperf/.env rename to hadoop-ozone/dist/src/main/compose/ozoneperf/.env diff --git a/hadoop-dist/src/main/compose/ozoneperf/README.md b/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md similarity index 100% rename from hadoop-dist/src/main/compose/ozoneperf/README.md rename to hadoop-ozone/dist/src/main/compose/ozoneperf/README.md diff --git a/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh b/hadoop-ozone/dist/src/main/compose/ozoneperf/compose-all.sh similarity index 100% rename from hadoop-dist/src/main/compose/ozoneperf/compose-all.sh rename to hadoop-ozone/dist/src/main/compose/ozoneperf/compose-all.sh diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose-freon.yaml similarity index 100% rename from hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml rename to hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose-freon.yaml diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml similarity index 98% rename from hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml rename to hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml index 3b5cc7b3b4..12b28bbcf0 100644 --- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml @@ -61,4 +61,4 @@ services: - "./prometheus.yml:/etc/prometheus.yml" command: ["--config.file","/etc/prometheus.yml"] ports: - - 9090:9090 + - 9090:9090 diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config similarity index 99% rename from hadoop-dist/src/main/compose/ozoneperf/docker-config rename to hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config index 454601e715..1ed116992c 100644 --- a/hadoop-dist/src/main/compose/ozoneperf/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config @@ -33,4 +33,3 @@ HADOOP_OPTS=-javaagent:/opt/jmxpromo.jar=port=0:consulHost=consul:consulMode=nod LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR - diff --git a/hadoop-dist/src/main/compose/ozoneperf/init.sh b/hadoop-ozone/dist/src/main/compose/ozoneperf/init.sh similarity index 100% rename from hadoop-dist/src/main/compose/ozoneperf/init.sh rename to hadoop-ozone/dist/src/main/compose/ozoneperf/init.sh diff --git a/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml b/hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml similarity index 100% rename from hadoop-dist/src/main/compose/ozoneperf/prometheus.yml rename to hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/authorized_keys similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys rename to hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/authorized_keys diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/config b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/config similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/.ssh/config rename to hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/config diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/environment similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/.ssh/environment rename to hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/environment diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa rename to hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub rename to hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub diff --git a/hadoop-dist/src/main/compose/ozonescripts/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/Dockerfile rename to hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile diff --git a/hadoop-dist/src/main/compose/ozonescripts/README.md b/hadoop-ozone/dist/src/main/compose/ozonescripts/README.md similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/README.md rename to hadoop-ozone/dist/src/main/compose/ozonescripts/README.md diff --git a/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml rename to hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml diff --git a/hadoop-dist/src/main/compose/ozonescripts/docker-config b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/docker-config rename to hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config diff --git a/hadoop-dist/src/main/compose/ozonescripts/ps.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/ps.sh rename to hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh diff --git a/hadoop-dist/src/main/compose/ozonescripts/start.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/start.sh rename to hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh diff --git a/hadoop-dist/src/main/compose/ozonescripts/stop.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh similarity index 100% rename from hadoop-dist/src/main/compose/ozonescripts/stop.sh rename to hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh diff --git a/hadoop-dist/src/main/ozone/README.txt b/hadoop-ozone/dist/src/main/ozone/README.txt similarity index 100% rename from hadoop-dist/src/main/ozone/README.txt rename to hadoop-ozone/dist/src/main/ozone/README.txt diff --git a/hadoop-dist/src/main/smoketest/README.md b/hadoop-ozone/dist/src/main/smoketest/README.md similarity index 100% rename from hadoop-dist/src/main/smoketest/README.md rename to hadoop-ozone/dist/src/main/smoketest/README.md diff --git a/hadoop-dist/src/main/smoketest/basic/basic.robot b/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot similarity index 100% rename from hadoop-dist/src/main/smoketest/basic/basic.robot rename to hadoop-ozone/dist/src/main/smoketest/basic/basic.robot diff --git a/hadoop-dist/src/main/smoketest/basic/ozone-shell.robot b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot similarity index 100% rename from hadoop-dist/src/main/smoketest/basic/ozone-shell.robot rename to hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot diff --git a/hadoop-dist/src/main/smoketest/commonlib.robot b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot similarity index 100% rename from hadoop-dist/src/main/smoketest/commonlib.robot rename to hadoop-ozone/dist/src/main/smoketest/commonlib.robot diff --git a/hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot similarity index 100% rename from hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot rename to hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot diff --git a/hadoop-dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh similarity index 100% rename from hadoop-dist/src/main/smoketest/test.sh rename to hadoop-ozone/dist/src/main/smoketest/test.sh diff --git a/hadoop-ozone/docs/pom.xml b/hadoop-ozone/docs/pom.xml index 64d0ec86e0..d8edd15e99 100644 --- a/hadoop-ozone/docs/pom.xml +++ b/hadoop-ozone/docs/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop Ozone Documentation jar - - ozone - true - - diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index 993e91f2c7..bed2fce653 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -28,37 +28,23 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop Ozone Integration Tests jar - - ozone - true - - org.apache.hadoop hadoop-ozone-common - provided org.apache.hadoop hadoop-ozone-ozone-manager - provided org.apache.hadoop hadoop-ozone-objectstore-service - provided org.apache.hadoop hadoop-ozone-client - provided - - - org.apache.hadoop - hadoop-ozone-tools - provided org.apache.hadoop diff --git a/hadoop-ozone/objectstore-service/pom.xml b/hadoop-ozone/objectstore-service/pom.xml index b20f1e239b..ce8fe4ccdd 100644 --- a/hadoop-ozone/objectstore-service/pom.xml +++ b/hadoop-ozone/objectstore-service/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop Ozone Object Store REST Service jar - - ozone - true - - diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 3efb33b10f..a3f69f6b92 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop Ozone Manager Server jar - - ozone - true - - @@ -61,28 +56,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - - org.apache.maven.plugins - maven-antrun-plugin - - - copy web resources - compile - - run - - - - - - - - - - - - - org.apache.maven.plugins maven-dependency-plugin @@ -93,19 +66,20 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> unpack + + + + org.apache.hadoop + hadoop-hdds-server-framework + ${project.build.outputDirectory} + + webapps/static/**/*.* + + + true + - - - - org.apache.hadoop - hadoop-hdds-server-framework - ${project.build.directory}/ - webapps/static/**/*.* - - - true - diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html similarity index 100% rename from hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html rename to hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css similarity index 100% rename from hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css rename to hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html similarity index 100% rename from hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html rename to hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html similarity index 100% rename from hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html rename to hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js similarity index 100% rename from hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js rename to hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml index c45aacd3c7..8174b74974 100644 --- a/hadoop-ozone/ozonefs/pom.xml +++ b/hadoop-ozone/ozonefs/pom.xml @@ -28,8 +28,6 @@ UTF-8 true - ozone - true diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index c73be60379..8d3e1613dd 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -19,9 +19,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 org.apache.hadoop - hadoop-project-dist + hadoop-project 3.2.0-SNAPSHOT - ../hadoop-project-dist + ../hadoop-project hadoop-ozone 0.3.0-SNAPSHOT @@ -42,6 +42,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> ozonefs integration-test objectstore-service + datanode + dist docs @@ -161,6 +163,25 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + org.apache.maven.plugins + maven-dependency-plugin + + + add-classpath-descriptor + package + + build-classpath + + + true + $HDDS_LIB_JARS_DIR + true + runtime + + + + diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index eeec595ee4..b04bbf1fe7 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -28,21 +28,33 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Hadoop Ozone Tools jar - - ozone - true - - org.apache.hadoop hadoop-ozone-common - provided org.apache.hadoop hadoop-ozone-client - provided + + + org.apache.hadoop + hadoop-ozone-filesystem + + + org.apache.hadoop + hadoop-common + compile + + + org.apache.hadoop + hadoop-hdfs + compile + + + org.apache.hadoop + hadoop-hdfs-client + compile io.dropwizard.metrics @@ -59,6 +71,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> jmh-generator-annprocess 1.19 + + org.apache.hadoop + hadoop-ozone-integration-test + test + test-jar + diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 794fdcc503..8d08703af9 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -616,6 +616,11 @@ hadoop-ozone-tools ${hdds.version} + + org.apache.hadoop + hadoop-ozone-datanode + ${hdds.version} + org.apache.hadoop hadoop-ozone-integration-test