From a34dafe325902164c04de0bf9d34eccdf6a2cd88 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Wed, 13 Aug 2014 17:30:26 +0000 Subject: [PATCH] HADOOP-8944. Shell command fs -count should include human readable option (Jonathan Allen via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617775 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/fs/ContentSummary.java | 48 +++++++++++++++---- .../org/apache/hadoop/fs/shell/Count.java | 36 ++++++++++++-- .../src/site/apt/FileSystemShell.apt.vm | 6 ++- .../src/test/resources/testConf.xml | 6 ++- .../src/test/resources/testHDFSConf.xml | 44 +++++++++++++++++ 6 files changed, 126 insertions(+), 17 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 06e9c25e35..547f2318fe 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -202,6 +202,9 @@ Trunk (Unreleased) HADOOP-10224. JavaKeyStoreProvider has to protect against corrupting underlying store. (asuresh via tucu) + HADOOP-8944. Shell command fs -count should include human readable option + (Jonathan Allen via aw) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java index 0d685b43e1..5d01637a0f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.util.StringUtils; /** Store the summary of a content (a directory or a file). */ @InterfaceAudience.Public @@ -102,7 +103,7 @@ public void readFields(DataInput in) throws IOException { * <----12----> <----12----> <-------18-------> * DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME */ - private static final String STRING_FORMAT = "%12d %12d %18d "; + private static final String STRING_FORMAT = "%12s %12s %18s "; /** * Output format: * <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18-------> @@ -117,7 +118,7 @@ public void readFields(DataInput in) throws IOException { private static final String QUOTA_HEADER = String.format( QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, - "quota", "remaining quota", "space quota", "reamaining quota") + + "name quota", "rem name quota", "space quota", "rem space quota") + HEADER; /** Return the header of the output. @@ -139,11 +140,25 @@ public String toString() { /** Return the string representation of the object in the output format. * if qOption is false, output directory count, file count, and content size; * if qOption is true, output quota and remaining quota as well. - * + * * @param qOption a flag indicating if quota needs to be printed or not * @return the string representation of the object - */ + */ public String toString(boolean qOption) { + return toString(qOption, false); + } + + /** Return the string representation of the object in the output format. + * if qOption is false, output directory count, file count, and content size; + * if qOption is true, output quota and remaining quota as well. + * if hOption is false file sizes are returned in bytes + * if hOption is true file sizes are returned in human readable + * + * @param qOption a flag indicating if quota needs to be printed or not + * @param hOption a flag indicating if human readable output if to be used + * @return the string representation of the object + */ + public String toString(boolean qOption, boolean hOption) { String prefix = ""; if (qOption) { String quotaStr = "none"; @@ -152,19 +167,32 @@ public String toString(boolean qOption) { String spaceQuotaRem = "inf"; if (quota>0) { - quotaStr = Long.toString(quota); - quotaRem = Long.toString(quota-(directoryCount+fileCount)); + quotaStr = formatSize(quota, hOption); + quotaRem = formatSize(quota-(directoryCount+fileCount), hOption); } if (spaceQuota>0) { - spaceQuotaStr = Long.toString(spaceQuota); - spaceQuotaRem = Long.toString(spaceQuota - spaceConsumed); + spaceQuotaStr = formatSize(spaceQuota, hOption); + spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption); } prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem); } - return prefix + String.format(STRING_FORMAT, directoryCount, - fileCount, length); + return prefix + String.format(STRING_FORMAT, + formatSize(directoryCount, hOption), + formatSize(fileCount, hOption), + formatSize(length, hOption)); + } + /** + * Formats a size to be human readable or in bytes + * @param size value to be formatted + * @param humanReadable flag indicating human readable or not + * @return String representation of the size + */ + private String formatSize(long size, boolean humanReadable) { + return humanReadable + ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1) + : String.valueOf(size); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index b8ccc0c9ec..ff7a10f297 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -42,16 +42,22 @@ public static void registerCommands(CommandFactory factory) { factory.addClass(Count.class, "-count"); } + private static final String OPTION_QUOTA = "q"; + private static final String OPTION_HUMAN = "h"; + public static final String NAME = "count"; - public static final String USAGE = "[-q] ..."; + public static final String USAGE = + "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] ..."; public static final String DESCRIPTION = "Count the number of directories, files and bytes under the paths\n" + "that match the specified file pattern. The output columns are:\n" + "DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" + "QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" + - " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME"; + " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" + + "The -h option shows file sizes in human readable format."; private boolean showQuotas; + private boolean humanReadable; /** Constructor */ public Count() {} @@ -70,17 +76,37 @@ public Count(String[] cmd, int pos, Configuration conf) { @Override protected void processOptions(LinkedList args) { - CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "q"); + CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, + OPTION_QUOTA, OPTION_HUMAN); cf.parse(args); if (args.isEmpty()) { // default path is the current working directory args.add("."); } - showQuotas = cf.getOpt("q"); + showQuotas = cf.getOpt(OPTION_QUOTA); + humanReadable = cf.getOpt(OPTION_HUMAN); } @Override protected void processPath(PathData src) throws IOException { ContentSummary summary = src.fs.getContentSummary(src.path); - out.println(summary.toString(showQuotas) + src); + out.println(summary.toString(showQuotas, isHumanReadable()) + src); + } + + /** + * Should quotas get shown as part of the report? + * @return if quotas should be shown then true otherwise false + */ + @InterfaceAudience.Private + boolean isShowQuotas() { + return showQuotas; + } + + /** + * Should sizes be shown in human readable format rather than bytes? + * @return true if human readable format + */ + @InterfaceAudience.Private + boolean isHumanReadable() { + return humanReadable; } } diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm index 4a82884fb3..48e0b21220 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm @@ -138,7 +138,7 @@ copyToLocal count - Usage: << >>> + Usage: << >>> Count the number of directories, files and bytes under the paths that match the specified file pattern. The output columns with -count are: DIR_COUNT, @@ -147,12 +147,16 @@ count The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA, REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, FILE_NAME + The -h option shows sizes in human readable format. + Example: * <<>> * <<>> + * <<>> + Exit Code: Returns 0 on success and -1 on error. diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index a91f0416d6..c44eddafe4 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -238,7 +238,7 @@ RegexpComparator - ^-count \[-q\] <path> \.\.\. :\s* + ^-count \[-q\] \[-h\] <path> \.\.\. :( )* RegexpComparator @@ -260,6 +260,10 @@ RegexpComparator ^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )* + + RegexpComparator + ^( |\t)*The -h option shows file sizes in human readable format.( )* + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 03083c50ce..087c3ab48a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -8655,6 +8655,50 @@ + + count: file using -h option + + -fs NAMENODE -mkdir -p dir + -fs NAMENODE -put CLITEST_DATA/data15bytes file1 + -fs NAMENODE -put CLITEST_DATA/data1k file2 + -fs NAMENODE -count -h file1 file2 + + + -fs NAMENODE -rm file1 file2 + + + + RegexpComparator + ( |\t)*0( |\t)*1( |\t)*15 file1 + + + + + RegexpComparator + ( |\t)*0( |\t)*1( |\t)*1\.0 K file2 + + + + + + count: directory using -q and -h options + + -fs NAMENODE -mkdir /dir1 + -fs NAMENODE -setQuota 10 /dir1 + -fs NAMENODE -setSpaceQuota 1m /dir1 + -fs NAMENODE -count -q -h /dir1 + + + -fs NAMENODE -rm -r /dir1 + + + + RegexpComparator + ( |\t)*10( |\t)*9( |\t)*1 M( |\t)*1 M( |\t)*1( |\t)*0( |\t)*0 /dir1 + + + + chmod: change permission(octal mode) of file in absolute path