diff --git a/common/CHANGES.txt b/common/CHANGES.txt index 5df8eeefa0..c30a638a67 100644 --- a/common/CHANGES.txt +++ b/common/CHANGES.txt @@ -296,7 +296,10 @@ Trunk (unreleased changes) HADOOP-7491. hadoop command should respect HADOOP_OPTS when given a class name. (eli) - + + HADOOP-7178. Add a parameter, useRawLocalFileSystem, to copyToLocalFile(..) + in FileSystem. (Uma Maheswara Rao G via szetszwo) + OPTIMIZATIONS HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole diff --git a/common/src/java/org/apache/hadoop/fs/FileSystem.java b/common/src/java/org/apache/hadoop/fs/FileSystem.java index 6395491096..d3fde9bb57 100644 --- a/common/src/java/org/apache/hadoop/fs/FileSystem.java +++ b/common/src/java/org/apache/hadoop/fs/FileSystem.java @@ -1707,7 +1707,38 @@ public void moveToLocalFile(Path src, Path dst) throws IOException { */ public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { - FileUtil.copy(this, src, getLocal(getConf()), dst, delSrc, getConf()); + copyToLocalFile(delSrc, src, dst, false); + } + + /** + * The src file is under FS, and the dst is on the local disk. Copy it from FS + * control to the local dst name. delSrc indicates if the src will be removed + * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem + * as local file system or not. RawLocalFileSystem is non crc file system.So, + * It will not create any crc files at local. + * + * @param delSrc + * whether to delete the src + * @param src + * path + * @param dst + * path + * @param useRawLocalFileSystem + * whether to use RawLocalFileSystem as local file system or not. + * + * @throws IOException + * - if any IO error + */ + public void copyToLocalFile(boolean delSrc, Path src, Path dst, + boolean useRawLocalFileSystem) throws IOException { + Configuration conf = getConf(); + FileSystem local = null; + if (useRawLocalFileSystem) { + local = getLocal(conf).getRawFileSystem(); + } else { + local = getLocal(conf); + } + FileUtil.copy(this, src, local, dst, delSrc, conf); } /** diff --git a/common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java b/common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java index 6a9079c8ec..9205f640ec 100644 --- a/common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java +++ b/common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java @@ -21,9 +21,11 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.URI; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsPermission; import org.junit.After; @@ -1080,6 +1082,31 @@ public void testGetWrappedInputStream() throws IOException { Assert.assertNotNull(is); } + @Test + public void testCopyToLocalWithUseRawLocalFileSystemOption() throws Exception { + Configuration conf = new Configuration(); + FileSystem fSys = new RawLocalFileSystem(); + Path fileToFS = new Path(TEST_ROOT_DIR, "fs.txt"); + Path fileToLFS = new Path(TEST_ROOT_DIR, "test.txt"); + Path crcFileAtLFS = new Path(TEST_ROOT_DIR, ".test.txt.crc"); + fSys.initialize(new URI("file:///"), conf); + writeFile(fSys, fileToFS); + if (fSys.exists(crcFileAtLFS)) + Assert.assertTrue("CRC files not deleted", fSys + .delete(crcFileAtLFS, true)); + fSys.copyToLocalFile(false, fileToFS, fileToLFS, true); + Assert.assertFalse("CRC files are created", fSys.exists(crcFileAtLFS)); + } + + private void writeFile(FileSystem fs, Path name) throws IOException { + FSDataOutputStream stm = fs.create(name); + try { + stm.writeBytes("42\n"); + } finally { + stm.close(); + } + } + protected void createFile(Path path) throws IOException { FileSystemTestHelper.createFile(fSys, path); }