From 18932717c42382ed8842de7719ec6d20c1765366 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Tue, 26 Jun 2018 18:28:47 -0700 Subject: [PATCH] HDDS-94. Change ozone datanode command to start the standalone datanode plugin. Contributed by Sandeep Nemuri. --- .../src/main/compose/ozone/docker-compose.yaml | 12 ------------ hadoop-dist/src/main/compose/ozone/docker-config | 5 ----- .../src/main/compose/ozoneperf/docker-compose.yaml | 13 ------------- .../src/main/compose/ozoneperf/docker-config | 5 ----- .../apache/hadoop/ozone/HddsDatanodeService.java | 13 ++++++++++++- .../src/test/acceptance/basic/docker-compose.yaml | 12 ------------ .../src/test/acceptance/basic/docker-config | 5 ----- .../src/test/acceptance/commonlib.robot | 1 - .../src/test/acceptance/ozonefs/docker-compose.yaml | 12 ------------ .../src/test/acceptance/ozonefs/docker-config | 5 ----- hadoop-ozone/common/src/main/bin/ozone | 9 +++------ 11 files changed, 15 insertions(+), 77 deletions(-) diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml index faf420c7f5..512c649e21 100644 --- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml @@ -16,18 +16,6 @@ version: "3" services: - namenode: - image: apache/hadoop-runner - hostname: namenode - volumes: - - ../../ozone:/opt/hadoop - ports: - - 9870:9870 - environment: - ENSURE_NAMENODE_DIR: /data/namenode - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/hdfs","namenode"] datanode: image: apache/hadoop-runner volumes: diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config index c693db0428..632f8701d2 100644 --- a/hadoop-dist/src/main/compose/ozone/docker-config +++ b/hadoop-dist/src/main/compose/ozone/docker-config @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 OZONE-SITE.XML_ozone.ksm.address=ksm OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.enabled=True @@ -23,12 +22,8 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 -HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml index fb7873bf88..3233c11641 100644 --- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml @@ -16,19 +16,6 @@ version: "3" services: - namenode: - image: apache/hadoop-runner - hostname: namenode - volumes: - - ../../ozone:/opt/hadoop - - ./jmxpromo.jar:/opt/jmxpromo.jar - ports: - - 9870:9870 - environment: - ENSURE_NAMENODE_DIR: /data/namenode - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/hdfs","namenode"] datanode: image: apache/hadoop-runner volumes: diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config index e4f5485ac5..2be22a7792 100644 --- a/hadoop-dist/src/main/compose/ozoneperf/docker-config +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-config @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 OZONE-SITE.XML_ozone.ksm.address=ksm OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.enabled=True @@ -23,12 +22,8 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 -HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index fa4187a254..ddeec873bc 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -25,9 +25,11 @@ import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.statemachine .DatanodeStateMachine; +import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.ServicePlugin; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; @@ -229,9 +231,18 @@ public static HddsDatanodeService createHddsDatanodeService( public static void main(String[] args) { try { + if (DFSUtil.parseHelpArgument(args, "Starts HDDS Datanode", System.out, false)) { + System.exit(0); + } + Configuration conf = new OzoneConfiguration(); + GenericOptionsParser hParser = new GenericOptionsParser(conf, args); + if (!hParser.isParseSuccessful()) { + GenericOptionsParser.printGenericCommandUsage(System.err); + System.exit(1); + } StringUtils.startupShutdownMessage(HddsDatanodeService.class, args, LOG); HddsDatanodeService hddsDatanodeService = - createHddsDatanodeService(new OzoneConfiguration()); + createHddsDatanodeService(conf); hddsDatanodeService.start(null); hddsDatanodeService.join(); } catch (Throwable e) { diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml index 44bd4a0aae..b50f42d3e9 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml @@ -16,18 +16,6 @@ version: "3" services: - namenode: - image: apache/hadoop-runner - hostname: namenode - volumes: - - ${OZONEDIR}:/opt/hadoop - ports: - - 9870 - environment: - ENSURE_NAMENODE_DIR: /data/namenode - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/hdfs","namenode"] datanode: image: apache/hadoop-runner volumes: diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config index 180dc8ef49..c3ec2ef71b 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 OZONE-SITE.XML_ozone.ksm.address=ksm OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874 OZONE-SITE.XML_ozone.scm.names=scm @@ -24,13 +23,9 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 -HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot index 01ed302e25..a5ea30af34 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot @@ -28,7 +28,6 @@ Startup Ozone cluster with size Daemons are running without error Is daemon running without error ksm Is daemon running without error scm - Is daemon running without error namenode Is daemon running without error datanode Check if datanode is connected to the scm diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml index 3323557511..12022dfe61 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml @@ -16,18 +16,6 @@ version: "3" services: - namenode: - image: apache/hadoop-runner - hostname: namenode - volumes: - - ${OZONEDIR}:/opt/hadoop - ports: - - 9870 - environment: - ENSURE_NAMENODE_DIR: /data/namenode - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/hdfs","namenode"] datanode: image: apache/hadoop-runner volumes: diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config index dec863e94e..e06d434bb4 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem OZONE-SITE.XML_ozone.ksm.address=ksm OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874 @@ -25,13 +24,9 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 -HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index 6843bddca5..390f0895b7 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -34,7 +34,7 @@ function hadoop_usage hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries" - hadoop_add_subcommand "datanode" daemon "run a DFS datanode" + hadoop_add_subcommand "datanode" daemon "run a HDDS datanode" hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables" hadoop_add_subcommand "freon" client "runs an ozone data generator" hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning." @@ -45,7 +45,7 @@ function hadoop_usage hadoop_add_subcommand "o3" client "command line interface for ozone" hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data" hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service" - hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager " + hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager" hadoop_add_subcommand "version" client "print the version" hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path" @@ -68,10 +68,7 @@ function ozonecmd_case ;; datanode) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - HADOOP_SECURE_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter" - HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode' - hadoop_deprecate_envvar HADOOP_SECURE_DN_PID_DIR HADOOP_SECURE_PID_DIR - hadoop_deprecate_envvar HADOOP_SECURE_DN_LOG_DIR HADOOP_SECURE_LOG_DIR + HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService ;; envvars) echo "JAVA_HOME='${JAVA_HOME}'"