From 2e4033e872331e08d032782ce2b39feb4a4d7e77 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Tue, 16 Sep 2014 17:52:28 -0700 Subject: [PATCH] HDFS-7052. Archival Storage: Add Mover into hdfs script. Contributed by Jing Zhao. --- .../hadoop-common/src/main/conf/hadoop-env.sh | 9 ++++++ .../hadoop-hdfs/src/main/bin/hdfs | 7 ++++ .../hadoop-hdfs/src/main/bin/hdfs.cmd | 8 ++++- .../hadoop/hdfs/server/mover/Mover.java | 32 +++++++++---------- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index eda47c9349..29ce8b1932 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -328,6 +328,15 @@ esac # # export HADOOP_BALANCER_OPTS="" +### +# HDFS Mover specific parameters +### +# Specify the JVM options to be used when starting the HDFS Mover. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HADOOP_MOVER_OPTS="" + ### # Advanced Users Only! ### diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 2ba6d1b1a9..22a0f0f8c0 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -36,6 +36,8 @@ function hadoop_usage echo " journalnode run the DFS journalnode" echo " lsSnapshottableDir list all snapshottable dirs owned by the current user" echo " Use -help to see options" + echo " mover run a utility to move block replicas across" + echo " storage types" echo " namenode run the DFS namenode" echo " Use -format to initialize the DFS filesystem" echo " nfs3 run an NFS version 3 gateway" @@ -159,6 +161,11 @@ case ${COMMAND} in lsSnapshottableDir) CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir ;; + mover) + CLASS=org.apache.hadoop.hdfs.server.mover.Mover + hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}" + ;; namenode) daemon="true" CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd index f5f77f043d..9fb84261e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd @@ -47,7 +47,7 @@ if "%1" == "--config" ( goto print_usage ) - set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin + set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover for %%i in ( %hdfscommands% ) do ( if %hdfs-command% == %%i set hdfscommand=true ) @@ -150,6 +150,11 @@ goto :eof set CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin goto :eof +:mover + set CLASS=org.apache.hadoop.hdfs.server.mover.Mover + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_MOVER_OPTS% + goto :eof + @rem This changes %1, %2 etc. Hence those cannot be used after calling this. :make_command_arguments if "%1" == "--config" ( @@ -198,6 +203,7 @@ goto :eof @echo lsSnapshottableDir list all snapshottable dirs owned by the current user @echo Use -help to see options @echo cacheadmin configure the HDFS cache + @echo mover run a utility to move block replicas across storage types @echo. @echo Most commands print help when invoked w/o parameters. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index e336ebc26b..0812c038a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -621,24 +621,24 @@ public int run(String[] args) throws Exception { System.out.println("Mover took " + StringUtils.formatTime(Time.monotonicNow()-startTime)); } } + } - /** - * Run a Mover in command line. - * - * @param args Command line arguments - */ - public static void main(String[] args) { - if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { - System.exit(0); - } + /** + * Run a Mover in command line. + * + * @param args Command line arguments + */ + public static void main(String[] args) { + if (DFSUtil.parseHelpArgument(args, Cli.USAGE, System.out, true)) { + System.exit(0); + } - try { - System.exit(ToolRunner.run(new HdfsConfiguration(), new Cli(), args)); - } catch (Throwable e) { - LOG.error("Exiting " + Mover.class.getSimpleName() - + " due to an exception", e); - System.exit(-1); - } + try { + System.exit(ToolRunner.run(new HdfsConfiguration(), new Cli(), args)); + } catch (Throwable e) { + LOG.error("Exiting " + Mover.class.getSimpleName() + + " due to an exception", e); + System.exit(-1); } } } \ No newline at end of file