big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16018. Optimize the display of hdfs "count -e" or "count -t" com… (#2994)

Browse Source
This commit is contained in:
Hongbing Wang 2021-05-20 11:23:54 +08:00 committed by GitHub
parent 3f5a66c158
commit f7247922b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 50 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
View File

@ -281,6 +281,20 @@ public int hashCode() {
private static final String ALL_HEADER = QUOTA_HEADER + SUMMARY_HEADER; private static final String ALL_HEADER = QUOTA_HEADER + SUMMARY_HEADER;
/**
* Output format:
* <--------20-------->
* ERASURECODING_POLICY
*/
private static final String ERASURECODING_POLICY_FORMAT = "%20s ";
private static final String ERASURECODING_POLICY_HEADER_FIELD =
"ERASURECODING_POLICY";
/** The header string. */
private static final String ERASURECODING_POLICY_HEADER = String.format(
ERASURECODING_POLICY_FORMAT, ERASURECODING_POLICY_HEADER_FIELD);
/** /**
* Output format:<-------18-------> <----------24----------> * Output format:<-------18-------> <----------24---------->
* <----------24---------->. <-------------28------------> SNAPSHOT_LENGTH * <----------24---------->. <-------------28------------> SNAPSHOT_LENGTH
@ -308,6 +322,10 @@ public static String getHeader(boolean qOption) {
return qOption ? ALL_HEADER : SUMMARY_HEADER; return qOption ? ALL_HEADER : SUMMARY_HEADER;
} }
public static String getErasureCodingPolicyHeader() {
return ERASURECODING_POLICY_HEADER;
}
public static String getSnapshotHeader() { public static String getSnapshotHeader() {
return SNAPSHOT_HEADER; return SNAPSHOT_HEADER;
} }
@ -444,6 +462,13 @@ private String formatSize(long size, boolean humanReadable) {
: String.valueOf(size); : String.valueOf(size);
} }
/**
* @return Constant-width String representation of Erasure Coding Policy
*/
public String toErasureCodingPolicy() {
return String.format(ERASURECODING_POLICY_FORMAT, erasureCodingPolicy);
}
/** /**
* Return the string representation of the snapshot counts in the output * Return the string representation of the snapshot counts in the output
* format. * format.

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
View File

@ -242,12 +242,14 @@ public boolean equals(Object obj) {
/** /**
* Output format: * Output format:
* |----12----| |------15-----| |------15-----| |------15-----| * |-----14-----| |-------18------| |-----14-----| |-------18------|
* QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA * SSD_QUOTA REM_SSD_QUOTA DISK_QUOTA REM_DISK_QUOTA
* |----12----| |----12----| |-------18-------| * |-----14-----| |-------18------| |-----14-----| |-------18------|
* DIR_COUNT FILE_COUNT CONTENT_SIZE * ARCHIVE_QUOTA REM_ARCHIVE_QUOTA PROVIDED_QUOTA REM_PROVIDED_QUOTA
* |-----14-----| |-------18------| |-------18------|
* NVDIMM_QUOTA REM_NVDIMM_QUOTA PATHNAME
*/ */
private static final String STORAGE_TYPE_SUMMARY_FORMAT = "%13s %17s "; private static final String STORAGE_TYPE_SUMMARY_FORMAT = "%14s %18s ";
/** Return the header of the output. /** Return the header of the output.
* @return the header of the output * @return the header of the output

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
View File

@ -166,8 +166,8 @@ protected void processOptions(LinkedList<String> args) {
headString.append(ContentSummary.getHeader(showQuotas)); headString.append(ContentSummary.getHeader(showQuotas));
} }
} }
if(displayECPolicy){ if (displayECPolicy) {
headString.append("ERASURECODING_POLICY "); headString.append(ContentSummary.getErasureCodingPolicyHeader());
} }
if (showSnapshot) { if (showSnapshot) {
headString.append(ContentSummary.getSnapshotHeader()); headString.append(ContentSummary.getSnapshotHeader());
@ -204,13 +204,9 @@ protected void processPath(PathData src) throws IOException {
outputString.append(summary.toString( outputString.append(summary.toString(
showQuotas, isHumanReadable(), excludeSnapshots)); showQuotas, isHumanReadable(), excludeSnapshots));
} }
if(displayECPolicy){ if (displayECPolicy) {
ContentSummary summary = src.fs.getContentSummary(src.path); ContentSummary summary = src.fs.getContentSummary(src.path);
if(!summary.getErasureCodingPolicy().equals("Replicated")){ outputString.append(summary.toErasureCodingPolicy());
outputString.append("EC:");
}
outputString.append(summary.getErasureCodingPolicy())
.append(" ");
} }
if (showSnapshot) { if (showSnapshot) {
ContentSummary summary = src.fs.getContentSummary(src.path); ContentSummary summary = src.fs.getContentSummary(src.path);

29
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
View File

@ -282,11 +282,10 @@ public void processPathWithQuotasByStorageTypesHeader() throws Exception {
options.add("dummy"); options.add("dummy");
count.processOptions(options); count.processOptions(options);
String withStorageTypeHeader = String withStorageTypeHeader =
// <----13---> <-------17------> <----13-----> <------17-------> // <----14----> <-------18-------> <-----14-----> <-------18------->
" SSD_QUOTA REM_SSD_QUOTA DISK_QUOTA REM_DISK_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA DISK_QUOTA REM_DISK_QUOTA " +
// <----13---> <-------17------> " ARCHIVE_QUOTA REM_ARCHIVE_QUOTA PROVIDED_QUOTA REM_PROVIDED_QUOTA " +
"ARCHIVE_QUOTA REM_ARCHIVE_QUOTA PROVIDED_QUOTA REM_PROVIDED_QUOTA " + " NVDIMM_QUOTA REM_NVDIMM_QUOTA " +
" NVDIMM_QUOTA REM_NVDIMM_QUOTA " +
"PATHNAME"; "PATHNAME";
verify(out).println(withStorageTypeHeader); verify(out).println(withStorageTypeHeader);
verifyNoMoreInteractions(out); verifyNoMoreInteractions(out);
@ -311,8 +310,8 @@ public void processPathWithQuotasBySSDStorageTypesHeader() throws Exception {
options.add("dummy"); options.add("dummy");
count.processOptions(options); count.processOptions(options);
String withStorageTypeHeader = String withStorageTypeHeader =
// <----13---> <-------17------> // <----14----> <-------18------->
" SSD_QUOTA REM_SSD_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA " +
"PATHNAME"; "PATHNAME";
verify(out).println(withStorageTypeHeader); verify(out).println(withStorageTypeHeader);
verifyNoMoreInteractions(out); verifyNoMoreInteractions(out);
@ -337,12 +336,12 @@ public void processPathWithQuotasByQTVH() throws Exception {
options.add("dummy"); options.add("dummy");
count.processOptions(options); count.processOptions(options);
String withStorageTypeHeader = String withStorageTypeHeader =
// <----13---> <-------17------> // <----14----> <-------18------->
" SSD_QUOTA REM_SSD_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA " +
" DISK_QUOTA REM_DISK_QUOTA " + " DISK_QUOTA REM_DISK_QUOTA " +
"ARCHIVE_QUOTA REM_ARCHIVE_QUOTA " + " ARCHIVE_QUOTA REM_ARCHIVE_QUOTA " +
"PROVIDED_QUOTA REM_PROVIDED_QUOTA " + "PROVIDED_QUOTA REM_PROVIDED_QUOTA " +
" NVDIMM_QUOTA REM_NVDIMM_QUOTA " + " NVDIMM_QUOTA REM_NVDIMM_QUOTA " +
"PATHNAME"; "PATHNAME";
verify(out).println(withStorageTypeHeader); verify(out).println(withStorageTypeHeader);
verifyNoMoreInteractions(out); verifyNoMoreInteractions(out);
@ -405,9 +404,9 @@ public void processPathWithQuotasByMultipleStorageTypes() throws Exception {
options.add("dummy"); options.add("dummy");
count.processOptions(options); count.processOptions(options);
String withStorageTypeHeader = String withStorageTypeHeader =
// <----13---> <------17-------> // <----14----> <------18-------->
" SSD_QUOTA REM_SSD_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA " +
" DISK_QUOTA REM_DISK_QUOTA " + " DISK_QUOTA REM_DISK_QUOTA " +
"PATHNAME"; "PATHNAME";
verify(out).println(withStorageTypeHeader); verify(out).println(withStorageTypeHeader);
verifyNoMoreInteractions(out); verifyNoMoreInteractions(out);