From c67b0650ea10896c6289703595faef0d262c00b3 Mon Sep 17 00:00:00 2001 From: Xiao Chen Date: Thu, 16 Aug 2018 23:13:10 -0700 Subject: [PATCH] HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi. --- .../java/org/apache/hadoop/hdfs/DistributedFileSystem.java | 6 +++++- .../org/apache/hadoop/hdfs/TestDistributedFileSystem.java | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 70b367927e..28c1e2735c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1217,7 +1217,11 @@ private DirListingIterator(Path p, PathFilter filter, thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME, needLocation); statistics.incrementReadOps(1); - storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + if (needLocation) { + storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + } else { + storageStatistics.incrementOpCounter(OpType.LIST_STATUS); + } if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index f09255e525..46323ddc33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -706,6 +706,7 @@ public void testStatistics() throws IOException { // Iterative ls test long mkdirOp = getOpStatistics(OpType.MKDIRS); long listStatusOp = getOpStatistics(OpType.LIST_STATUS); + long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS); for (int i = 0; i < 10; i++) { Path p = new Path(dir, Integer.toString(i)); fs.mkdirs(p); @@ -729,6 +730,12 @@ public void testStatistics() throws IOException { checkStatistics(fs, readOps, ++writeOps, largeReadOps); checkOpStatistics(OpType.MKDIRS, mkdirOp); checkOpStatistics(OpType.LIST_STATUS, listStatusOp); + + fs.listLocatedStatus(dir); + locatedListStatusOP++; + readOps++; + checkStatistics(fs, readOps, writeOps, largeReadOps); + checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP); } opCount = getOpStatistics(OpType.GET_STATUS);