From a2b1ff0257bde26d1f64454e97bc1225294a30b9 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Mon, 28 Nov 2016 11:54:43 -0800 Subject: [PATCH] HADOOP-13828. Implement getFileChecksum(path, length) for ViewFileSystem. Contributed by Manoj Govindassamy. --- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 6 ++++ .../hadoop/fs/viewfs/ViewFileSystem.java | 9 ++++++ .../fs/viewfs/TestViewFileSystemHdfs.java | 29 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index 9f61af6955..272433f450 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -221,6 +221,12 @@ public FileChecksum getFileChecksum(final Path f) return super.getFileChecksum(fullPath(f)); } + @Override + public FileChecksum getFileChecksum(final Path f, final long length) + throws IOException { + return super.getFileChecksum(fullPath(f), length); + } + @Override public FileStatus getFileStatus(final Path f) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 9061b2ae12..ed1bda2d54 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -350,6 +350,15 @@ public FileChecksum getFileChecksum(final Path f) return res.targetFileSystem.getFileChecksum(res.remainingPath); } + @Override + public FileChecksum getFileChecksum(final Path f, final long length) + throws AccessControlException, FileNotFoundException, + IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(f), true); + return res.targetFileSystem.getFileChecksum(res.remainingPath, length); + } + private static FileStatus fixFileStatus(FileStatus orig, Path qualified) throws IOException { // FileStatus#getPath is a fully qualified path relative to the root of diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index 0e420d0d68..58b77f6f91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -31,6 +31,8 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FsConstants; @@ -218,4 +220,31 @@ public void testDf() throws Exception { DFSTestUtil.FsShellRun("-df /", 0, null, newConf); DFSTestUtil.FsShellRun("-df", 0, null, newConf); } + + @Test + public void testFileChecksum() throws IOException { + ViewFileSystem viewFs = (ViewFileSystem) fsView; + Path mountDataRootPath = new Path("/data"); + String fsTargetFileName = "debug.log"; + Path fsTargetFilePath = new Path(targetTestRoot, "data/debug.log"); + Path mountDataFilePath = new Path(mountDataRootPath, fsTargetFileName); + + fileSystemTestHelper.createFile(fsTarget, fsTargetFilePath); + FileStatus fileStatus = viewFs.getFileStatus(mountDataFilePath); + long fileLength = fileStatus.getLen(); + + FileChecksum fileChecksumViaViewFs = + viewFs.getFileChecksum(mountDataFilePath); + FileChecksum fileChecksumViaTargetFs = + fsTarget.getFileChecksum(fsTargetFilePath); + Assert.assertTrue("File checksum not matching!", + fileChecksumViaViewFs.equals(fileChecksumViaTargetFs)); + + fileChecksumViaViewFs = + viewFs.getFileChecksum(mountDataFilePath, fileLength / 2); + fileChecksumViaTargetFs = + fsTarget.getFileChecksum(fsTargetFilePath, fileLength / 2); + Assert.assertTrue("File checksum not matching!", + fileChecksumViaViewFs.equals(fileChecksumViaTargetFs)); + } }