big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi.

Browse Source
This commit is contained in:
Xiao Chen 2018-08-16 23:13:10 -07:00
parent 1697a02306
commit c67b0650ea
2 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -1217,7 +1217,11 @@ private DirListingIterator(Path p, PathFilter filter,
thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME, thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME,
needLocation); needLocation);
statistics.incrementReadOps(1); statistics.incrementReadOps(1);
if (needLocation) {
storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS);
} else {
storageStatistics.incrementOpCounter(OpType.LIST_STATUS);
}
if (thisListing == null) { // the directory does not exist if (thisListing == null) { // the directory does not exist
throw new FileNotFoundException("File " + p + " does not exist."); throw new FileNotFoundException("File " + p + " does not exist.");
} }

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
View File

@ -706,6 +706,7 @@ public void testStatistics() throws IOException {
// Iterative ls test // Iterative ls test
long mkdirOp = getOpStatistics(OpType.MKDIRS); long mkdirOp = getOpStatistics(OpType.MKDIRS);
long listStatusOp = getOpStatistics(OpType.LIST_STATUS); long listStatusOp = getOpStatistics(OpType.LIST_STATUS);
long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS);
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
Path p = new Path(dir, Integer.toString(i)); Path p = new Path(dir, Integer.toString(i));
fs.mkdirs(p); fs.mkdirs(p);
@ -729,6 +730,12 @@ public void testStatistics() throws IOException {
checkStatistics(fs, readOps, ++writeOps, largeReadOps); checkStatistics(fs, readOps, ++writeOps, largeReadOps);
checkOpStatistics(OpType.MKDIRS, mkdirOp); checkOpStatistics(OpType.MKDIRS, mkdirOp);
checkOpStatistics(OpType.LIST_STATUS, listStatusOp); checkOpStatistics(OpType.LIST_STATUS, listStatusOp);
fs.listLocatedStatus(dir);
locatedListStatusOP++;
readOps++;
checkStatistics(fs, readOps, writeOps, largeReadOps);
checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP);
} }
opCount = getOpStatistics(OpType.GET_STATUS); opCount = getOpStatistics(OpType.GET_STATUS);