From 374ddd9f9ea43b0e730a7baab3e51e6893d40420 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 29 May 2015 17:15:58 -0700 Subject: [PATCH] HADOOP-12043. Display warning if defaultFs is not set when running fs commands. Contributed by Eddy Xu. --- .../hadoop-common/CHANGES.txt | 3 ++ .../fs/CommonConfigurationKeysPublic.java | 6 ++++ .../org/apache/hadoop/fs/shell/FsCommand.java | 30 ++++++++++++++++++- .../java/org/apache/hadoop/fs/shell/Ls.java | 8 +++++ .../src/main/resources/core-default.xml | 8 +++++ .../org/apache/hadoop/fs/shell/TestLs.java | 25 ++++++++++++++++ 6 files changed, 79 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e5d2e390ef..00075b404c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating (Masatake Iwasaki via Colin P. McCabe) + HADOOP-12043. Display warning if defaultFs is not set when running fs + commands. (Lei Xu via wang) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 90c69345f5..7231d59924 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -363,5 +363,11 @@ public class CommonConfigurationKeysPublic { "hadoop.security.random.device.file.path"; public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT = "/dev/urandom"; + + /** See core-default.xml */ + public static final String HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY = + "hadoop.shell.missing.defaultFs.warning"; + public static final boolean HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT = + false; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java index 9515fde4c4..971950f755 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.shell; import java.io.IOException; +import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -27,8 +28,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.shell.find.Find; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY; + /** - * Base class for all "hadoop fs" commands + * Base class for all "hadoop fs" commands. */ @InterfaceAudience.Private @@ -90,4 +96,26 @@ protected void run(Path path) throws IOException { public int runAll() { return run(args); } + + @Override + protected void processRawArguments(LinkedList args) + throws IOException { + LinkedList expendedArgs = expandArguments(args); + // If "fs.defaultFs" is not set appropriately, it warns the user that the + // command is not running against HDFS. + final boolean displayWarnings = getConf().getBoolean( + HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, + HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT); + if (displayWarnings) { + final String defaultFs = getConf().get(FS_DEFAULT_NAME_KEY); + final boolean missingDefaultFs = + defaultFs == null || defaultFs.equals(FS_DEFAULT_NAME_DEFAULT); + if (missingDefaultFs) { + err.printf( + "Warning: fs.defaultFs is not set when running \"%s\" command.%n", + getCommandName()); + } + } + processArguments(expendedArgs); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java index d5c52eda9d..67348c6c78 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java @@ -24,6 +24,8 @@ import java.util.Comparator; import java.util.Date; import java.util.LinkedList; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; @@ -102,6 +104,12 @@ public static void registerCommands(CommandFactory factory) { protected boolean humanReadable = false; + protected Ls() {} + + protected Ls(Configuration conf) { + super(conf); + } + protected String formatSize(long size) { return humanReadable ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index a1bc780649..ae23a750a9 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1899,4 +1899,12 @@ for ldap providers in the same way as above does. Client + + + Enable hdfs shell commands to display warnings if (fs.defaultFS) property + is not set. + + hadoop.shell.missing.defaultFs.warning + false + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java index 4a9103ffc1..cba44ee117 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.fs.shell; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY; import static org.junit.Assert.*; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.*; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; import java.net.URI; @@ -1048,6 +1050,29 @@ public void processPathDirectoryPathOnly() throws IOException { verifyNoMoreInteractions(out); } + private static void displayWarningOnLocalFileSystem(boolean shouldDisplay) + throws IOException { + Configuration conf = new Configuration(); + conf.setBoolean( + HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, shouldDisplay); + + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(buf, true); + Ls ls = new Ls(conf); + ls.err = err; + ls.run("file:///."); + assertEquals(shouldDisplay, buf.toString().contains( + "Warning: fs.defaultFs is not set when running \"ls\" command.")); + } + + @Test + public void displayWarningsOnLocalFileSystem() throws IOException { + // Display warnings. + displayWarningOnLocalFileSystem(true); + // Does not display warnings. + displayWarningOnLocalFileSystem(false); + } + // check the deprecated flag isn't set @Test public void isDeprecated() {