big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-10971. Add -C flag to make hadoop fs -ls print filenames only. Contributed by Kengo Seki.

Browse Source
This commit is contained in:
Akira Ajisaka 2015-05-19 17:25:27 +09:00
parent 93972a332a
commit 3b50dcdce4
5 changed files with 106 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -488,6 +488,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11949. Add user-provided plugins to test-patch (Sean Busbey via aw)
HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only.
(Kengo Seki via aajisaka)
IMPROVEMENTS
HADOOP-6842. "hadoop fs -text" does not give a useful text representation

32
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
View File

@ -43,6 +43,7 @@ public static void registerCommands(CommandFactory factory) {
factory.addClass(Lsr.class, "-lsr");
}
private static final String OPTION_PATHONLY = "C";
private static final String OPTION_DIRECTORY = "d";
private static final String OPTION_HUMAN = "h";
private static final String OPTION_RECURSIVE = "R";
@ -52,10 +53,10 @@ public static void registerCommands(CommandFactory factory) {
private static final String OPTION_SIZE = "S";
public static final String NAME = "ls";
public static final String USAGE = "[-" + OPTION_DIRECTORY + "] [-"
+ OPTION_HUMAN + "] " + "[-" + OPTION_RECURSIVE + "] [-" + OPTION_MTIME
+ "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] " + "[-"
+ OPTION_ATIME + "] [<path> ...]";
public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-"
+ OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" + OPTION_RECURSIVE
+ "] [-" + OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE
+ "] [-" + OPTION_ATIME + "] [<path> ...]";
public static final String DESCRIPTION =
"List the contents that match the specified file pattern. If " +
@ -67,6 +68,8 @@ public static void registerCommands(CommandFactory factory) {
"\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" +
"and file entries are of the form:\n" +
"\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n\n" +
" -" + OPTION_PATHONLY +
" Display the paths of files and directories only.\n" +
" -" + OPTION_DIRECTORY +
" Directories are listed as plain files.\n" +
" -" + OPTION_HUMAN +
@ -89,6 +92,7 @@ public static void registerCommands(CommandFactory factory) {
protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
protected String lineFormat;
private boolean pathOnly;
protected boolean dirRecurse;
private boolean orderReverse;
private boolean orderTime;
@ -107,10 +111,11 @@ protected String formatSize(long size) {
@Override
protected void processOptions(LinkedList<String> args)
throws IOException {
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, OPTION_PATHONLY,
OPTION_DIRECTORY, OPTION_HUMAN, OPTION_RECURSIVE, OPTION_REVERSE,
OPTION_MTIME, OPTION_SIZE, OPTION_ATIME);
cf.parse(args);
pathOnly = cf.getOpt(OPTION_PATHONLY);
dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
setRecursive(cf.getOpt(OPTION_RECURSIVE) && dirRecurse);
humanReadable = cf.getOpt(OPTION_HUMAN);
@ -123,6 +128,15 @@ protected void processOptions(LinkedList<String> args)
initialiseOrderComparator();
}
/**
* Should display only paths of files and directories.
* @return true display paths only, false display all fields
*/
@InterfaceAudience.Private
boolean isPathOnly() {
return this.pathOnly;
}
/**
* Should the contents of the directory be shown or just the directory?
* @return true if directory contents, false if just directory
@ -191,15 +205,23 @@ protected void processPathArgument(PathData item) throws IOException {
protected void processPaths(PathData parent, PathData ... items)
throws IOException {
if (parent != null && !isRecursive() && items.length != 0) {
if (!pathOnly) {
out.println("Found " + items.length + " items");
}
Arrays.sort(items, getOrderComparator());
}
if (!pathOnly) {
adjustColumnWidths(items);
}
super.processPaths(parent, items);
}
@Override
protected void processPath(PathData item) throws IOException {
if (pathOnly) {
out.println(item.toString());
return;
}
FileStatus stat = item.stat;
String line = String.format(lineFormat,
(stat.isDirectory() ? "d" : "-"),

3
hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
View File

@ -382,10 +382,11 @@ Return usage output.
ls
----
Usage: `hadoop fs -ls [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
Usage: `hadoop fs -ls [-C] [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
Options:
* -C: Display the paths of files and directories only.
* -d: Directories are listed as plain files.
* -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864).
* -R: Recursively list subdirectories encountered.

67
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
View File

@ -74,6 +74,24 @@ public void processOptionsNone() throws IOException {
LinkedList<String> options = new LinkedList<String>();
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
assertFalse(ls.isOrderReverse());
assertFalse(ls.isOrderSize());
assertFalse(ls.isOrderTime());
assertFalse(ls.isUseAtime());
}
// check the -C option is recognised
@Test
public void processOptionsPathOnly() throws IOException {
LinkedList<String> options = new LinkedList<String>();
options.add("-C");
Ls ls = new Ls();
ls.processOptions(options);
assertTrue(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@ -90,6 +108,7 @@ public void processOptionsDirectory() throws IOException {
options.add("-d");
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertFalse(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@ -106,6 +125,7 @@ public void processOptionsHuman() throws IOException {
options.add("-h");
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertTrue(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@ -122,6 +142,7 @@ public void processOptionsRecursive() throws IOException {
options.add("-R");
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertTrue(ls.isRecursive());
@ -138,6 +159,7 @@ public void processOptionsReverse() throws IOException {
options.add("-r");
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@ -154,6 +176,7 @@ public void processOptionsSize() throws IOException {
options.add("-S");
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@ -170,6 +193,7 @@ public void processOptionsMtime() throws IOException {
options.add("-t");
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@ -187,6 +211,7 @@ public void processOptionsMtimeSize() throws IOException {
options.add("-S");
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@ -205,6 +230,7 @@ public void processOptionsMtimeSizeReverse() throws IOException {
options.add("-r");
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@ -221,6 +247,7 @@ public void processOptionsAtime() throws IOException {
options.add("-u");
Ls ls = new Ls();
ls.processOptions(options);
assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@ -234,6 +261,7 @@ public void processOptionsAtime() throws IOException {
@Test
public void processOptionsAll() throws IOException {
LinkedList<String> options = new LinkedList<String>();
options.add("-C"); // show file path only
options.add("-d"); // directory
options.add("-h"); // human readable
options.add("-R"); // recursive
@ -243,6 +271,7 @@ public void processOptionsAll() throws IOException {
options.add("-u"); // show atime
Ls ls = new Ls();
ls.processOptions(options);
assertTrue(ls.isPathOnly());
assertFalse(ls.isDirRecurse());
assertTrue(ls.isHumanReadable());
assertFalse(ls.isRecursive()); // -d overrules -R
@ -981,6 +1010,44 @@ public void processPathDirOrderAtimeReverse() throws IOException {
verifyNoMoreInteractions(out);
}
// check path only display (-C option)
@Test
public void processPathDirectoryPathOnly() throws IOException {
TestFile testfile01 = new TestFile("testDirectory", "testFile01");
TestFile testfile02 = new TestFile("testDirectory", "testFile02");
TestFile testfile03 = new TestFile("testDirectory", "testFile03");
TestFile testfile04 = new TestFile("testDirectory", "testFile04");
TestFile testfile05 = new TestFile("testDirectory", "testFile05");
TestFile testfile06 = new TestFile("testDirectory", "testFile06");
TestFile testDir = new TestFile("", "testDirectory");
testDir.setIsDir(true);
testDir.addContents(testfile01, testfile02, testfile03, testfile04,
testfile05, testfile06);
LinkedList<PathData> pathData = new LinkedList<PathData>();
pathData.add(testDir.getPathData());
PrintStream out = mock(PrintStream.class);
Ls ls = new Ls();
ls.out = out;
LinkedList<String> options = new LinkedList<String>();
options.add("-C");
ls.processOptions(options);
ls.processArguments(pathData);
InOrder inOrder = inOrder(out);
inOrder.verify(out).println(testfile01.getPath().toString());
inOrder.verify(out).println(testfile02.getPath().toString());
inOrder.verify(out).println(testfile03.getPath().toString());
inOrder.verify(out).println(testfile04.getPath().toString());
inOrder.verify(out).println(testfile05.getPath().toString());
inOrder.verify(out).println(testfile06.getPath().toString());
verifyNoMoreInteractions(out);
}
// check the deprecated flag isn't set
@Test
public void isDeprecated() {

6
hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
View File

@ -54,7 +54,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-ls \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[&lt;path&gt; \.\.\.\] :( |\t)*</expected-output>
<expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[&lt;path&gt; \.\.\.\] :( |\t)*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@ -92,6 +92,10 @@
<type>RegexpComparator</type>
<expected-output>^\s*modificationDate\(yyyy-MM-dd HH:mm\) fileName( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^\s*-C\s+Display the paths of files and directories only\.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^\s*-d\s+Directories are listed as plain files\.( )*</expected-output>