HADOOP-10971. Add -C flag to make hadoop fs -ls print filenames only. Contributed by Kengo Seki.

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -488,6 +488,9 @@ Release 2.8.0 - UNRELEASED
 
     HADOOP-11949. Add user-provided plugins to test-patch (Sean Busbey via aw)
 
+    HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only.
+    (Kengo Seki via aajisaka)
+
   IMPROVEMENTS
 
     HADOOP-6842. "hadoop fs -text" does not give a useful text representation

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java

@@ -43,6 +43,7 @@ public static void registerCommands(CommandFactory factory) {
     factory.addClass(Lsr.class, "-lsr");
   }
 
+  private static final String OPTION_PATHONLY = "C";
   private static final String OPTION_DIRECTORY = "d";
   private static final String OPTION_HUMAN = "h";
   private static final String OPTION_RECURSIVE = "R";
@@ -52,10 +53,10 @@ public static void registerCommands(CommandFactory factory) {
   private static final String OPTION_SIZE = "S";
 
   public static final String NAME = "ls";
-  public static final String USAGE = "[-" + OPTION_DIRECTORY + "] [-"
+  public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-"
-      + OPTION_HUMAN + "] " + "[-" + OPTION_RECURSIVE + "] [-" + OPTION_MTIME
+      + OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" + OPTION_RECURSIVE
-      + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] " + "[-"
+      + "] [-" + OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE
-      + OPTION_ATIME + "] [<path> ...]";
+      + "] [-" + OPTION_ATIME + "] [<path> ...]";
 
   public static final String DESCRIPTION =
       "List the contents that match the specified file pattern. If " +
@@ -67,6 +68,8 @@ public static void registerCommands(CommandFactory factory) {
           "\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" +
           "and file entries are of the form:\n" +
           "\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n\n" +
+          "  -" + OPTION_PATHONLY +
+          "  Display the paths of files and directories only.\n" +
           "  -" + OPTION_DIRECTORY +
           "  Directories are listed as plain files.\n" +
           "  -" + OPTION_HUMAN +
@@ -89,6 +92,7 @@ public static void registerCommands(CommandFactory factory) {
 
   protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
   protected String lineFormat;
+  private boolean pathOnly;
   protected boolean dirRecurse;
   private boolean orderReverse;
   private boolean orderTime;
@@ -107,10 +111,11 @@ protected String formatSize(long size) {
   @Override
   protected void processOptions(LinkedList<String> args)
   throws IOException {
-    CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
+    CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, OPTION_PATHONLY,
         OPTION_DIRECTORY, OPTION_HUMAN, OPTION_RECURSIVE, OPTION_REVERSE,
         OPTION_MTIME, OPTION_SIZE, OPTION_ATIME);
     cf.parse(args);
+    pathOnly = cf.getOpt(OPTION_PATHONLY);
     dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
     setRecursive(cf.getOpt(OPTION_RECURSIVE) && dirRecurse);
     humanReadable = cf.getOpt(OPTION_HUMAN);
@@ -123,6 +128,15 @@ protected void processOptions(LinkedList<String> args)
     initialiseOrderComparator();
   }
 
+  /**
+   * Should display only paths of files and directories.
+   * @return true display paths only, false display all fields
+   */
+  @InterfaceAudience.Private
+  boolean isPathOnly() {
+    return this.pathOnly;
+  }
+
   /**
    * Should the contents of the directory be shown or just the directory?
    * @return true if directory contents, false if just directory
@@ -191,15 +205,23 @@ protected void processPathArgument(PathData item) throws IOException {
   protected void processPaths(PathData parent, PathData ... items)
   throws IOException {
     if (parent != null && !isRecursive() && items.length != 0) {
-      out.println("Found " + items.length + " items");
+      if (!pathOnly) {
+        out.println("Found " + items.length + " items");
+      }
       Arrays.sort(items, getOrderComparator());
     }
-    adjustColumnWidths(items);
+    if (!pathOnly) {
+      adjustColumnWidths(items);
+    }
     super.processPaths(parent, items);
   }
 
   @Override
   protected void processPath(PathData item) throws IOException {
+    if (pathOnly) {
+      out.println(item.toString());
+      return;
+    }
     FileStatus stat = item.stat;
     String line = String.format(lineFormat,
         (stat.isDirectory() ? "d" : "-"),

hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md

@@ -382,10 +382,11 @@ Return usage output.
 ls
 ----
 
-Usage: `hadoop fs -ls [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
+Usage: `hadoop fs -ls [-C] [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
 
 Options:
 
+* -C: Display the paths of files and directories only.
 * -d: Directories are listed as plain files.
 * -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864).
 * -R: Recursively list subdirectories encountered.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java

@@ -74,6 +74,24 @@ public void processOptionsNone() throws IOException {
     LinkedList<String> options = new LinkedList<String>();
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
+    assertTrue(ls.isDirRecurse());
+    assertFalse(ls.isHumanReadable());
+    assertFalse(ls.isRecursive());
+    assertFalse(ls.isOrderReverse());
+    assertFalse(ls.isOrderSize());
+    assertFalse(ls.isOrderTime());
+    assertFalse(ls.isUseAtime());
+  }
+
+  // check the -C option is recognised
+  @Test
+  public void processOptionsPathOnly() throws IOException {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-C");
+    Ls ls = new Ls();
+    ls.processOptions(options);
+    assertTrue(ls.isPathOnly());
     assertTrue(ls.isDirRecurse());
     assertFalse(ls.isHumanReadable());
     assertFalse(ls.isRecursive());
@@ -90,6 +108,7 @@ public void processOptionsDirectory() throws IOException {
     options.add("-d");
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
     assertFalse(ls.isDirRecurse());
     assertFalse(ls.isHumanReadable());
     assertFalse(ls.isRecursive());
@@ -106,6 +125,7 @@ public void processOptionsHuman() throws IOException {
     options.add("-h");
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
     assertTrue(ls.isDirRecurse());
     assertTrue(ls.isHumanReadable());
     assertFalse(ls.isRecursive());
@@ -122,6 +142,7 @@ public void processOptionsRecursive() throws IOException {
     options.add("-R");
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
     assertTrue(ls.isDirRecurse());
     assertFalse(ls.isHumanReadable());
     assertTrue(ls.isRecursive());
@@ -138,6 +159,7 @@ public void processOptionsReverse() throws IOException {
     options.add("-r");
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
     assertTrue(ls.isDirRecurse());
     assertFalse(ls.isHumanReadable());
     assertFalse(ls.isRecursive());
@@ -154,6 +176,7 @@ public void processOptionsSize() throws IOException {
     options.add("-S");
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
     assertTrue(ls.isDirRecurse());
     assertFalse(ls.isHumanReadable());
     assertFalse(ls.isRecursive());
@@ -170,6 +193,7 @@ public void processOptionsMtime() throws IOException {
     options.add("-t");
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
     assertTrue(ls.isDirRecurse());
     assertFalse(ls.isHumanReadable());
     assertFalse(ls.isRecursive());
@@ -187,6 +211,7 @@ public void processOptionsMtimeSize() throws IOException {
     options.add("-S");
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
     assertTrue(ls.isDirRecurse());
     assertFalse(ls.isHumanReadable());
     assertFalse(ls.isRecursive());
@@ -205,6 +230,7 @@ public void processOptionsMtimeSizeReverse() throws IOException {
     options.add("-r");
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
     assertTrue(ls.isDirRecurse());
     assertFalse(ls.isHumanReadable());
     assertFalse(ls.isRecursive());
@@ -221,6 +247,7 @@ public void processOptionsAtime() throws IOException {
     options.add("-u");
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertFalse(ls.isPathOnly());
     assertTrue(ls.isDirRecurse());
     assertFalse(ls.isHumanReadable());
     assertFalse(ls.isRecursive());
@@ -234,6 +261,7 @@ public void processOptionsAtime() throws IOException {
   @Test
   public void processOptionsAll() throws IOException {
     LinkedList<String> options = new LinkedList<String>();
+    options.add("-C"); // show file path only
     options.add("-d"); // directory
     options.add("-h"); // human readable
     options.add("-R"); // recursive
@@ -243,6 +271,7 @@ public void processOptionsAll() throws IOException {
     options.add("-u"); // show atime
     Ls ls = new Ls();
     ls.processOptions(options);
+    assertTrue(ls.isPathOnly());
     assertFalse(ls.isDirRecurse());
     assertTrue(ls.isHumanReadable());
     assertFalse(ls.isRecursive()); // -d overrules -R
@@ -981,6 +1010,44 @@ public void processPathDirOrderAtimeReverse() throws IOException {
     verifyNoMoreInteractions(out);
   }
 
+  // check path only display (-C option)
+  @Test
+  public void processPathDirectoryPathOnly() throws IOException {
+    TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+    TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+    TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+    TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+    TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+    TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+    TestFile testDir = new TestFile("", "testDirectory");
+    testDir.setIsDir(true);
+    testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+        testfile05, testfile06);
+
+    LinkedList<PathData> pathData = new LinkedList<PathData>();
+    pathData.add(testDir.getPathData());
+
+    PrintStream out = mock(PrintStream.class);
+
+    Ls ls = new Ls();
+    ls.out = out;
+
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-C");
+    ls.processOptions(options);
+
+    ls.processArguments(pathData);
+    InOrder inOrder = inOrder(out);
+    inOrder.verify(out).println(testfile01.getPath().toString());
+    inOrder.verify(out).println(testfile02.getPath().toString());
+    inOrder.verify(out).println(testfile03.getPath().toString());
+    inOrder.verify(out).println(testfile04.getPath().toString());
+    inOrder.verify(out).println(testfile05.getPath().toString());
+    inOrder.verify(out).println(testfile06.getPath().toString());
+    verifyNoMoreInteractions(out);
+  }
+
   // check the deprecated flag isn't set
   @Test
   public void isDeprecated() {

hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

@@ -54,7 +54,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-ls \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[&lt;path&gt; \.\.\.\] :( |\t)*</expected-output>
+          <expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[&lt;path&gt; \.\.\.\] :( |\t)*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -92,6 +92,10 @@
           <type>RegexpComparator</type>
           <expected-output>^\s*modificationDate\(yyyy-MM-dd HH:mm\) fileName( )*</expected-output>
         </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-C\s+Display the paths of files and directories only\.( )*</expected-output>
+        </comparator>
         <comparator>
           <type>RegexpComparator</type>
           <expected-output>^\s*-d\s+Directories are listed as plain files\.( )*</expected-output>