big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-12043. Display warning if defaultFs is not set when running fs commands. Contributed by Eddy Xu.

Browse Source
This commit is contained in:
Andrew Wang 2015-05-29 17:15:58 -07:00
parent 6aec13cb33
commit 374ddd9f9e
6 changed files with 79 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating
(Masatake Iwasaki via Colin P. McCabe)
HADOOP-12043. Display warning if defaultFs is not set when running fs
commands. (Lei Xu via wang)
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
View File

@ -363,5 +363,11 @@ public class CommonConfigurationKeysPublic {
"hadoop.security.random.device.file.path";
public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT =
"/dev/urandom";
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY =
"hadoop.shell.missing.defaultFs.warning";
public static final boolean HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT =
false;
}

30
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.fs.shell;
import java.io.IOException;
import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -27,8 +28,13 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.find.Find;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
/**
* Base class for all "hadoop fs" commands
* Base class for all "hadoop fs" commands.
*/
@InterfaceAudience.Private
@ -90,4 +96,26 @@ protected void run(Path path) throws IOException {
public int runAll() {
return run(args);
}
@Override
protected void processRawArguments(LinkedList<String> args)
throws IOException {
LinkedList<PathData> expendedArgs = expandArguments(args);
// If "fs.defaultFs" is not set appropriately, it warns the user that the
// command is not running against HDFS.
final boolean displayWarnings = getConf().getBoolean(
HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY,
HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT);
if (displayWarnings) {
final String defaultFs = getConf().get(FS_DEFAULT_NAME_KEY);
final boolean missingDefaultFs =
defaultFs == null || defaultFs.equals(FS_DEFAULT_NAME_DEFAULT);
if (missingDefaultFs) {
err.printf(
"Warning: fs.defaultFs is not set when running \"%s\" command.%n",
getCommandName());
}
}
processArguments(expendedArgs);
}
}

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
View File

@ -24,6 +24,8 @@
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@ -102,6 +104,12 @@ public static void registerCommands(CommandFactory factory) {
protected boolean humanReadable = false;
protected Ls() {}
protected Ls(Configuration conf) {
super(conf);
}
protected String formatSize(long size) {
return humanReadable
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)

8
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -1899,4 +1899,12 @@ for ldap providers in the same way as above does.
<value>Client</value>
</property>
<property>
<description>
Enable hdfs shell commands to display warnings if (fs.defaultFS) property
is not set.
</description>
<name>hadoop.shell.missing.defaultFs.warning</name>
<value>false</value>
</property>
</configuration>

25
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
View File

@ -17,10 +17,12 @@
*/
package org.apache.hadoop.fs.shell;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
import static org.junit.Assert.*;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.*;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
@ -1048,6 +1050,29 @@ public void processPathDirectoryPathOnly() throws IOException {
verifyNoMoreInteractions(out);
}
private static void displayWarningOnLocalFileSystem(boolean shouldDisplay)
throws IOException {
Configuration conf = new Configuration();
conf.setBoolean(
HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, shouldDisplay);
ByteArrayOutputStream buf = new ByteArrayOutputStream();
PrintStream err = new PrintStream(buf, true);
Ls ls = new Ls(conf);
ls.err = err;
ls.run("file:///.");
assertEquals(shouldDisplay, buf.toString().contains(
"Warning: fs.defaultFs is not set when running \"ls\" command."));
}
@Test
public void displayWarningsOnLocalFileSystem() throws IOException {
// Display warnings.
displayWarningOnLocalFileSystem(true);
// Does not display warnings.
displayWarningOnLocalFileSystem(false);
}
// check the deprecated flag isn't set
@Test
public void isDeprecated() {