big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16954. Add -S option in "Count" command to show only Snapshot Counts. Contributed by hemanthboyina.

Browse Source 
(cherry picked from commit b89d875f7b)
This commit is contained in:
Takanobu Asanuma 2020-04-06 11:03:10 +09:00 committed by S O'Donnell
parent e944cc0338
commit 65bf544118
6 changed files with 110 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
View File

@ -281,6 +281,21 @@ public int hashCode() {
private static final String ALL_HEADER = QUOTA_HEADER + SUMMARY_HEADER;
/**
* Output format:<-------18-------> <----------24---------->
* <----------24---------->. <-------------28------------> SNAPSHOT_LENGTH
* SNAPSHOT_FILE_COUNT SNAPSHOT_DIR_COUNT SNAPSHOT_SPACE_CONSUMED
*/
private static final String SNAPSHOT_FORMAT = "%18s %24s %24s %28s ";
private static final String[] SNAPSHOT_HEADER_FIELDS =
new String[] {"SNAPSHOT_LENGTH", "SNAPSHOT_FILE_COUNT",
"SNAPSHOT_DIR_COUNT", "SNAPSHOT_SPACE_CONSUMED"};
/** The header string. */
private static final String SNAPSHOT_HEADER =
String.format(SNAPSHOT_FORMAT, (Object[]) SNAPSHOT_HEADER_FIELDS);
/** Return the header of the output.
* if qOption is false, output directory count, file count, and content size;
@ -293,7 +308,9 @@ public static String getHeader(boolean qOption) {
return qOption ? ALL_HEADER : SUMMARY_HEADER;
}
public static String getSnapshotHeader() {
return SNAPSHOT_HEADER;
}
/**
* Returns the names of the fields from the summary header.
@ -416,7 +433,7 @@ public String toString(boolean qOption, boolean hOption, boolean tOption,
}
/**
* Formats a size to be human readable or in bytes
* Formats a size to be human readable or in bytes.
* @param size value to be formatted
* @param humanReadable flag indicating human readable or not
* @return String representation of the size
@ -426,4 +443,17 @@ private String formatSize(long size, boolean humanReadable) {
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
: String.valueOf(size);
}
/**
* Return the string representation of the snapshot counts in the output
* format.
* @param hOption flag indicating human readable or not
* @return String representation of the snapshot counts
*/
public String toSnapshot(boolean hOption) {
return String.format(SNAPSHOT_FORMAT, formatSize(snapshotLength, hOption),
formatSize(snapshotFileCount, hOption),
formatSize(snapshotDirectoryCount, hOption),
formatSize(snapshotSpaceConsumed, hOption));
}
}

17
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
View File

@ -56,13 +56,14 @@ public static void registerCommands(CommandFactory factory) {
//return the quota, namespace count and disk space usage.
private static final String OPTION_QUOTA_AND_USAGE = "u";
private static final String OPTION_ECPOLICY = "e";
private static final String OPTION_SNAPSHOT_COUNT = "s";
public static final String NAME = "count";
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
+ "] [-" + OPTION_ECPOLICY
+ "] [-" + OPTION_ECPOLICY + "] [-" + OPTION_SNAPSHOT_COUNT
+ "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
@ -93,7 +94,8 @@ public static void registerCommands(CommandFactory factory) {
"the storage types.\n" +
"The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and \n" +
"the usage against the quota without the detailed content summary."+
"The -"+ OPTION_ECPOLICY +" option shows the erasure coding policy.";
"The -" + OPTION_ECPOLICY + " option shows the erasure coding policy."
+ "The -" + OPTION_SNAPSHOT_COUNT + " option shows snapshot counts.";
private boolean showQuotas;
private boolean humanReadable;
@ -102,6 +104,7 @@ public static void registerCommands(CommandFactory factory) {
private boolean showQuotasAndUsageOnly;
private boolean excludeSnapshots;
private boolean displayECPolicy;
private boolean showSnapshot;
/** Constructor */
public Count() {}
@ -123,7 +126,7 @@ protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
OPTION_EXCLUDE_SNAPSHOT,
OPTION_ECPOLICY);
OPTION_ECPOLICY, OPTION_SNAPSHOT_COUNT);
cf.addOptionWithValue(OPTION_TYPE);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
@ -134,6 +137,7 @@ protected void processOptions(LinkedList<String> args) {
showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
displayECPolicy = cf.getOpt(OPTION_ECPOLICY);
showSnapshot = cf.getOpt(OPTION_SNAPSHOT_COUNT);
if (showQuotas || showQuotasAndUsageOnly) {
String types = cf.getOptValue(OPTION_TYPE);
@ -165,6 +169,9 @@ protected void processOptions(LinkedList<String> args) {
if(displayECPolicy){
headString.append("ERASURECODING_POLICY ");
}
if (showSnapshot) {
headString.append(ContentSummary.getSnapshotHeader());
}
headString.append("PATHNAME");
out.println(headString.toString());
}
@ -205,6 +212,10 @@ protected void processPath(PathData src) throws IOException {
outputString.append(summary.getErasureCodingPolicy())
.append(" ");
}
if (showSnapshot) {
ContentSummary summary = src.fs.getContentSummary(src.path);
outputString.append(summary.toSnapshot(isHumanReadable()));
}
outputString.append(src);
out.println(outputString.toString());
}

5
hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
View File

@ -132,7 +132,7 @@ Identical to the -get command.
count
-----
Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e] <paths> `
Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e] [-s] <paths> `
Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
@ -156,6 +156,8 @@ The output columns with -count -e are: DIR\_COUNT, FILE\_COUNT, CONTENT_SIZE, ER
The ERASURECODING\_POLICY is name of the policy for the file. If a erasure coding policy is setted on that file, it will return name of the policy. If no erasure coding policy is setted, it will return \"Replicated\" which means it use replication storage strategy.
The -s option shows the snapshot counts for each directory.
Example:
* `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@ -166,6 +168,7 @@ Example:
* `hadoop fs -count -u -h hdfs://nn1.example.com/file1`
* `hadoop fs -count -u -h -v hdfs://nn1.example.com/file1`
* `hadoop fs -count -e hdfs://nn1.example.com/file1`
* `hadoop fs -count -s hdfs://nn1.example.com/file1`
Exit Code:

36
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
View File

@ -253,4 +253,40 @@ public void testToStringHumanNoShowQuota() {
String expected = " 32.6 K 211.9 M 8.0 E ";
assertEquals(expected, contentSummary.toString(false, true));
}
// check the toSnapshot method with human readable.
@Test
public void testToSnapshotHumanReadable() {
long snapshotLength = Long.MAX_VALUE;
long snapshotFileCount = 222222222;
long snapshotDirectoryCount = 33333;
long snapshotSpaceConsumed = 222256578;
ContentSummary contentSummary = new ContentSummary.Builder()
.snapshotLength(snapshotLength).snapshotFileCount(snapshotFileCount)
.snapshotDirectoryCount(snapshotDirectoryCount)
.snapshotSpaceConsumed(snapshotSpaceConsumed).build();
String expected =
" 8.0 E 211.9 M 32.6 K "
+ " 212.0 M ";
assertEquals(expected, contentSummary.toSnapshot(true));
}
// check the toSnapshot method with human readable disabled.
@Test
public void testToSnapshotNotHumanReadable() {
long snapshotLength = 1111;
long snapshotFileCount = 2222;
long snapshotDirectoryCount = 3333;
long snapshotSpaceConsumed = 4444;
ContentSummary contentSummary = new ContentSummary.Builder()
.snapshotLength(snapshotLength).snapshotFileCount(snapshotFileCount)
.snapshotDirectoryCount(snapshotDirectoryCount)
.snapshotSpaceConsumed(snapshotSpaceConsumed).build();
String expected =
" 1111 2222 3333 "
+ " 4444 ";
assertEquals(expected, contentSummary.toSnapshot(false));
}
}

25
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
View File

@ -411,6 +411,25 @@ public void processPathWithQuotasByMultipleStorageTypes() throws Exception {
verifyNoMoreInteractions(out);
}
@Test
public void processPathWithSnapshotHeader() throws Exception {
Path path = new Path("mockfs:/test");
when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat);
PrintStream out = mock(PrintStream.class);
Count count = new Count();
count.out = out;
LinkedList<String> options = new LinkedList<String>();
options.add("-s");
options.add("-v");
options.add("dummy");
count.processOptions(options);
String withSnapshotHeader = " DIR_COUNT FILE_COUNT CONTENT_SIZE "
+ " SNAPSHOT_LENGTH SNAPSHOT_FILE_COUNT "
+ " SNAPSHOT_DIR_COUNT SNAPSHOT_SPACE_CONSUMED PATHNAME";
verify(out).println(withSnapshotHeader);
verifyNoMoreInteractions(out);
}
@Test
public void getCommandName() {
Count count = new Count();
@ -448,7 +467,8 @@ public void getUsage() {
Count count = new Count();
String actual = count.getUsage();
String expected =
"-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] [-e] <path> ...";
"-count [-q] [-h] [-v] [-t [<storage type>]]"
+ " [-u] [-x] [-e] [-s] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
@ -480,7 +500,8 @@ public void getDescription() {
+ "storage types.\n"
+ "The -u option shows the quota and \n"
+ "the usage against the quota without the detailed content summary."
+ "The -e option shows the erasure coding policy.";
+ "The -e option shows the erasure coding policy."
+ "The -s option shows snapshot counts.";
assertEquals("Count.getDescription", expected, actual);
}

2
hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
View File

@ -282,7 +282,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] \[-e\] &lt;path&gt; \.\.\. :( )*</expected-output>
<expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] \[-e\] \[-s\] &lt;path&gt; \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>