diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 53655d1687..d585bbc94b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -819,6 +819,8 @@ public BlockStoragePolicySpi next(final FileSystem fs, final Path p) @Override public Collection getAllStoragePolicies() throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICIES); return Arrays.asList(dfs.getStoragePolicies()); } @@ -840,9 +842,7 @@ public long getBytesWithFutureGenerationStamps() throws IOException { */ @Deprecated public BlockStoragePolicy[] getStoragePolicies() throws IOException { - statistics.incrementReadOps(1); - storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICIES); - return dfs.getStoragePolicies(); + return getAllStoragePolicies().toArray(new BlockStoragePolicy[0]); } /** @@ -2162,6 +2162,9 @@ public Void next(final FileSystem fs, final Path p) */ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() throws IOException { + statistics.incrementReadOps(1); + storageStatistics + .incrementOpCounter(OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST); return dfs.getSnapshottableDirListing(); } @@ -2334,6 +2337,8 @@ private SnapshotDiffReport getSnapshotDiffReportInternal( */ public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir, final String fromSnapshot, final String toSnapshot) throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF); Path absF = fixRelativePart(snapshotDir); return new FileSystemLinkResolver() { @Override @@ -3282,6 +3287,8 @@ public ECTopologyVerifierResult getECTopologyResultForPolicies( */ @Override public Path getTrashRoot(Path path) { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOT); try { if ((path == null) || !dfs.isHDFSEncryptionEnabled()) { return super.getTrashRoot(path); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index ca59cf396e..7a0857e2e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -1341,6 +1341,8 @@ public void allowSnapshot(final Path p) throws IOException { @Override public void satisfyStoragePolicy(final Path p) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.SATISFY_STORAGE_POLICY); final HttpOpParam.Op op = PutOpParam.Op.SATISFYSTORAGEPOLICY; new FsPathRunner(op, p).run(); } @@ -1430,6 +1432,7 @@ public void renameSnapshot(final Path path, final String snapshotOldName, public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir, final String fromSnapshot, final String toSnapshot) throws IOException { + statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF); final HttpOpParam.Op op = GetOpParam.Op.GETSNAPSHOTDIFF; return new FsPathResponseRunner(op, snapshotDir, @@ -1444,6 +1447,7 @@ SnapshotDiffReport decodeResponse(Map json) { public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList() throws IOException { + statistics.incrementReadOps(1); storageStatistics .incrementOpCounter(OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST); final HttpOpParam.Op op = GetOpParam.Op.GETSNAPSHOTTABLEDIRECTORYLIST; @@ -2005,6 +2009,8 @@ public void setStoragePolicy(Path p, String policyName) throws IOException { @Override public Collection getAllStoragePolicies() throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICIES); final HttpOpParam.Op op = GetOpParam.Op.GETALLSTORAGEPOLICY; return new FsPathResponseRunner>(op, null) { @Override @@ -2017,6 +2023,8 @@ Collection decodeResponse(Map json) @Override public BlockStoragePolicy getStoragePolicy(Path src) throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICY); final HttpOpParam.Op op = GetOpParam.Op.GETSTORAGEPOLICY; return new FsPathResponseRunner(op, src) { @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 3c903563f4..e96bd624e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -920,6 +920,21 @@ public void testStatistics2() throws IOException, NoSuchAlgorithmException { dfs.getEZForPath(dir); checkStatistics(dfs, ++readOps, writeOps, 0); checkOpStatistics(OpType.GET_ENCRYPTION_ZONE, opCount + 1); + + opCount = getOpStatistics(OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST); + dfs.getSnapshottableDirListing(); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST, opCount + 1); + + opCount = getOpStatistics(OpType.GET_STORAGE_POLICIES); + dfs.getAllStoragePolicies(); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_STORAGE_POLICIES, opCount + 1); + + opCount = getOpStatistics(OpType.GET_TRASH_ROOT); + dfs.getTrashRoot(dir); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_TRASH_ROOT, opCount + 1); } } @@ -1060,7 +1075,7 @@ public void run() { } /** Checks statistics. -1 indicates do not check for the operations */ - private void checkStatistics(FileSystem fs, int readOps, int writeOps, + public static void checkStatistics(FileSystem fs, int readOps, int writeOps, int largeReadOps) { assertEquals(readOps, DFSTestUtil.getStatistics(fs).getReadOps()); assertEquals(writeOps, DFSTestUtil.getStatistics(fs).getWriteOps()); @@ -1166,12 +1181,12 @@ private void testReadFileSystemStatistics(int expectedDistance, } } - private static void checkOpStatistics(OpType op, long count) { + public static void checkOpStatistics(OpType op, long count) { assertEquals("Op " + op.getSymbol() + " has unexpected count!", count, getOpStatistics(op)); } - private static long getOpStatistics(OpType op) { + public static long getOpStatistics(OpType op) { return GlobalStorageStatistics.INSTANCE.get( DFSOpsCountStatistics.NAME) .getLong(op.getSymbol()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index d02a5fec4d..27a69855ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -24,6 +24,9 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; +import static org.apache.hadoop.hdfs.TestDistributedFileSystem.checkOpStatistics; +import static org.apache.hadoop.hdfs.TestDistributedFileSystem.checkStatistics; +import static org.apache.hadoop.hdfs.TestDistributedFileSystem.getOpStatistics; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY; import static org.junit.Assert.assertEquals; @@ -58,6 +61,7 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.QuotaUsage; +import org.apache.hadoop.hdfs.DFSOpsCountStatistics; import org.apache.hadoop.test.LambdaTestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -2013,6 +2017,62 @@ public void testECPolicyInFileStatus() throws Exception { ecpolicyForECfile, ecPolicyName); } + @Test + public void testStatistics() throws Exception { + final Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY, + StoragePolicySatisfierMode.EXTERNAL.toString()); + StoragePolicySatisfier sps = new StoragePolicySatisfier(conf); + try { + cluster = new MiniDFSCluster.Builder(conf).storageTypes( + new StorageType[][] {{StorageType.DISK, StorageType.ARCHIVE}}) + .storagesPerDatanode(2).numDataNodes(1).build(); + cluster.waitActive(); + sps.init(new ExternalSPSContext(sps, DFSTestUtil + .getNameNodeConnector(conf, HdfsServerConstants.MOVER_ID_PATH, 1, + false))); + sps.start(StoragePolicySatisfierMode.EXTERNAL); + sps.start(StoragePolicySatisfierMode.EXTERNAL); + final WebHdfsFileSystem webHdfs = WebHdfsTestUtil + .getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME); + Path dir = new Path("/test"); + webHdfs.mkdirs(dir); + int readOps = 0; + int writeOps = 0; + FileSystem.clearStatistics(); + + long opCount = + getOpStatistics(DFSOpsCountStatistics.OpType.GET_STORAGE_POLICY); + webHdfs.getStoragePolicy(dir); + checkStatistics(webHdfs, ++readOps, writeOps, 0); + checkOpStatistics(DFSOpsCountStatistics.OpType.GET_STORAGE_POLICY, + opCount + 1); + + opCount = + getOpStatistics(DFSOpsCountStatistics.OpType.GET_STORAGE_POLICIES); + webHdfs.getAllStoragePolicies(); + checkStatistics(webHdfs, ++readOps, writeOps, 0); + checkOpStatistics(DFSOpsCountStatistics.OpType.GET_STORAGE_POLICIES, + opCount + 1); + + opCount = + getOpStatistics(DFSOpsCountStatistics.OpType.SATISFY_STORAGE_POLICY); + webHdfs.satisfyStoragePolicy(dir); + checkStatistics(webHdfs, readOps, ++writeOps, 0); + checkOpStatistics(DFSOpsCountStatistics.OpType.SATISFY_STORAGE_POLICY, + opCount + 1); + + opCount = getOpStatistics( + DFSOpsCountStatistics.OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST); + webHdfs.getSnapshottableDirectoryList(); + checkStatistics(webHdfs, ++readOps, writeOps, 0); + checkOpStatistics( + DFSOpsCountStatistics.OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST, + opCount + 1); + } finally { + cluster.shutdown(); + } + } /** * Get FileStatus JSONObject from ListStatus response. */