big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17999. No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem. Contributed by Abhishek Das. (#3639)

Browse Source 
(cherry picked from commit 54a1d78e16)
This commit is contained in:
Abhishek Das 2021-11-16 17:56:30 -08:00 committed by Konstantin V Shvachko
parent 026d5860cb
commit f456dc1837
3 changed files with 49 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -917,13 +917,9 @@ public void removeXAttr(Path path, String name) throws IOException {
} }
@Override @Override
public void setVerifyChecksum(final boolean verifyChecksum) { public void setVerifyChecksum(final boolean verifyChecksum) {
List<InodeTree.MountPoint<FileSystem>> mountPoints = // This is a file system level operations, however ViewFileSystem
fsState.getMountPoints(); // points to many file systems. Noop for ViewFileSystem.
Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints);
for (InodeTree.MountPoint<FileSystem> mount : mountPoints) {
fsMap.get(mount.src).setVerifyChecksum(verifyChecksum);
}
} }
/** /**
@ -1019,13 +1015,9 @@ public QuotaUsage getQuotaUsage(Path f) throws IOException {
} }
@Override @Override
public void setWriteChecksum(final boolean writeChecksum) { public void setWriteChecksum(final boolean writeChecksum) {
List<InodeTree.MountPoint<FileSystem>> mountPoints = // This is a file system level operations, however ViewFileSystem
fsState.getMountPoints(); // points to many file systems. Noop for ViewFileSystem.
Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints);
for (InodeTree.MountPoint<FileSystem> mount : mountPoints) {
fsMap.get(mount.src).setWriteChecksum(writeChecksum);
}
} }
@Override @Override

12
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java
View File

@ -83,12 +83,6 @@ public void testSanity() throws URISyntaxException {
assertEquals(new URI("fs2:/").getAuthority(), fs2.getUri().getAuthority()); assertEquals(new URI("fs2:/").getAuthority(), fs2.getUri().getAuthority());
} }
@Test
public void testVerifyChecksum() throws Exception {
checkVerifyChecksum(false);
checkVerifyChecksum(true);
}
/** /**
* Tests that ViewFileSystem dispatches calls for every ACL method through the * Tests that ViewFileSystem dispatches calls for every ACL method through the
* mount table to the correct underlying FileSystem with all Path arguments * mount table to the correct underlying FileSystem with all Path arguments
@ -144,12 +138,6 @@ public void testAclMethods() throws Exception {
verify(mockFs2).getAclStatus(mockFsPath2); verify(mockFs2).getAclStatus(mockFsPath2);
} }
void checkVerifyChecksum(boolean flag) {
viewFs.setVerifyChecksum(flag);
assertEquals(flag, fs1.getVerifyChecksum());
assertEquals(flag, fs2.getVerifyChecksum());
}
static class FakeFileSystem extends LocalFileSystem { static class FakeFileSystem extends LocalFileSystem {
boolean verifyChecksum = true; boolean verifyChecksum = true;
URI uri; URI uri;

43
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
View File

@ -1474,4 +1474,47 @@ public void testTargetFileSystemLazyInitialization() throws Exception {
// viewfs inner cache is disabled // viewfs inner cache is disabled
assertEquals(cacheSize + 2, TestFileUtil.getCacheSize()); assertEquals(cacheSize + 2, TestFileUtil.getCacheSize());
} }
@Test
public void testTargetFileSystemLazyInitializationForChecksumMethods()
throws Exception {
final String clusterName = "cluster" + new Random().nextInt();
Configuration config = new Configuration(conf);
config.setBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE, false);
config.setClass("fs.othermockfs.impl",
TestChRootedFileSystem.MockFileSystem.class, FileSystem.class);
ConfigUtil.addLink(config, clusterName, "/user",
URI.create("othermockfs://mockauth1/mockpath"));
ConfigUtil.addLink(config, clusterName,
"/mock", URI.create("othermockfs://mockauth/mockpath"));
final int cacheSize = TestFileUtil.getCacheSize();
ViewFileSystem viewFs = (ViewFileSystem) FileSystem.get(
new URI("viewfs://" + clusterName + "/"), config);
// As no inner file system instance has been initialized,
// cache size will remain the same
// cache is disabled for viewfs scheme, so the viewfs:// instance won't
// go in the cache even after the initialization
assertEquals(cacheSize, TestFileUtil.getCacheSize());
// This is not going to initialize any filesystem instance
viewFs.setVerifyChecksum(true);
// Cache size will remain the same
assertEquals(cacheSize, TestFileUtil.getCacheSize());
// This resolve path will initialize the file system corresponding
// to the mount table entry of the path "/user"
viewFs.getFileChecksum(
new Path(String.format("viewfs://%s/%s", clusterName, "/user")));
// Cache size will increase by 1.
assertEquals(cacheSize + 1, TestFileUtil.getCacheSize());
viewFs.close();
// Initialized FileSystem instances will not be removed from cache as
// viewfs inner cache is disabled
assertEquals(cacheSize + 1, TestFileUtil.getCacheSize());
}
} }