From 54a1d78e16533e286455de62a545ee75cbc1eff5 Mon Sep 17 00:00:00 2001 From: Abhishek Das Date: Tue, 16 Nov 2021 17:56:30 -0800 Subject: [PATCH] HADOOP-17999. No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem. Contributed by Abhishek Das. (#3639) --- .../hadoop/fs/viewfs/ViewFileSystem.java | 20 +++------ .../viewfs/TestViewFileSystemDelegation.java | 12 ------ .../fs/viewfs/ViewFileSystemBaseTest.java | 43 +++++++++++++++++++ 3 files changed, 49 insertions(+), 26 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index ce918a13ea..538f03a130 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -917,13 +917,9 @@ public void removeXAttr(Path path, String name) throws IOException { } @Override - public void setVerifyChecksum(final boolean verifyChecksum) { - List> mountPoints = - fsState.getMountPoints(); - Map fsMap = initializeMountedFileSystems(mountPoints); - for (InodeTree.MountPoint mount : mountPoints) { - fsMap.get(mount.src).setVerifyChecksum(verifyChecksum); - } + public void setVerifyChecksum(final boolean verifyChecksum) { + // This is a file system level operations, however ViewFileSystem + // points to many file systems. Noop for ViewFileSystem. } /** @@ -1019,13 +1015,9 @@ public QuotaUsage getQuotaUsage(Path f) throws IOException { } @Override - public void setWriteChecksum(final boolean writeChecksum) { - List> mountPoints = - fsState.getMountPoints(); - Map fsMap = initializeMountedFileSystems(mountPoints); - for (InodeTree.MountPoint mount : mountPoints) { - fsMap.get(mount.src).setWriteChecksum(writeChecksum); - } + public void setWriteChecksum(final boolean writeChecksum) { + // This is a file system level operations, however ViewFileSystem + // points to many file systems. Noop for ViewFileSystem. } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java index d8c39f79d0..3a60d6ecdd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java @@ -83,12 +83,6 @@ public void testSanity() throws URISyntaxException { assertEquals(new URI("fs2:/").getAuthority(), fs2.getUri().getAuthority()); } - @Test - public void testVerifyChecksum() throws Exception { - checkVerifyChecksum(false); - checkVerifyChecksum(true); - } - /** * Tests that ViewFileSystem dispatches calls for every ACL method through the * mount table to the correct underlying FileSystem with all Path arguments @@ -144,12 +138,6 @@ public void testAclMethods() throws Exception { verify(mockFs2).getAclStatus(mockFsPath2); } - void checkVerifyChecksum(boolean flag) { - viewFs.setVerifyChecksum(flag); - assertEquals(flag, fs1.getVerifyChecksum()); - assertEquals(flag, fs2.getVerifyChecksum()); - } - static class FakeFileSystem extends LocalFileSystem { boolean verifyChecksum = true; URI uri; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index be50f457be..434eff0bef 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -1472,4 +1472,47 @@ public void testTargetFileSystemLazyInitialization() throws Exception { // viewfs inner cache is disabled assertEquals(cacheSize + 2, TestFileUtil.getCacheSize()); } + + @Test + public void testTargetFileSystemLazyInitializationForChecksumMethods() + throws Exception { + final String clusterName = "cluster" + new Random().nextInt(); + Configuration config = new Configuration(conf); + config.setBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); + config.setClass("fs.othermockfs.impl", + TestChRootedFileSystem.MockFileSystem.class, FileSystem.class); + ConfigUtil.addLink(config, clusterName, "/user", + URI.create("othermockfs://mockauth1/mockpath")); + ConfigUtil.addLink(config, clusterName, + "/mock", URI.create("othermockfs://mockauth/mockpath")); + + final int cacheSize = TestFileUtil.getCacheSize(); + ViewFileSystem viewFs = (ViewFileSystem) FileSystem.get( + new URI("viewfs://" + clusterName + "/"), config); + + // As no inner file system instance has been initialized, + // cache size will remain the same + // cache is disabled for viewfs scheme, so the viewfs:// instance won't + // go in the cache even after the initialization + assertEquals(cacheSize, TestFileUtil.getCacheSize()); + + // This is not going to initialize any filesystem instance + viewFs.setVerifyChecksum(true); + + // Cache size will remain the same + assertEquals(cacheSize, TestFileUtil.getCacheSize()); + + // This resolve path will initialize the file system corresponding + // to the mount table entry of the path "/user" + viewFs.getFileChecksum( + new Path(String.format("viewfs://%s/%s", clusterName, "/user"))); + + // Cache size will increase by 1. + assertEquals(cacheSize + 1, TestFileUtil.getCacheSize()); + + viewFs.close(); + // Initialized FileSystem instances will not be removed from cache as + // viewfs inner cache is disabled + assertEquals(cacheSize + 1, TestFileUtil.getCacheSize()); + } }