big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-7178. Add a parameter, useRawLocalFileSystem, to copyToLocalFile(..) in FileSystem. Contributed by Uma Maheswara Rao G

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1152791 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2011-08-01 14:20:24 +00:00
parent 7e18c90d39
commit 54860d86ab
3 changed files with 63 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
common/CHANGES.txt
View File

@ -296,7 +296,10 @@ Trunk (unreleased changes)
HADOOP-7491. hadoop command should respect HADOOP_OPTS when given
a class name. (eli)
HADOOP-7178. Add a parameter, useRawLocalFileSystem, to copyToLocalFile(..)
in FileSystem. (Uma Maheswara Rao G via szetszwo)
OPTIMIZATIONS
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole

33
common/src/java/org/apache/hadoop/fs/FileSystem.java
View File

@ -1707,7 +1707,38 @@ public void moveToLocalFile(Path src, Path dst) throws IOException {
*/
public void copyToLocalFile(boolean delSrc, Path src, Path dst)
throws IOException {
FileUtil.copy(this, src, getLocal(getConf()), dst, delSrc, getConf());
copyToLocalFile(delSrc, src, dst, false);
}
/**
* The src file is under FS, and the dst is on the local disk. Copy it from FS
* control to the local dst name. delSrc indicates if the src will be removed
* or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
* as local file system or not. RawLocalFileSystem is non crc file system.So,
* It will not create any crc files at local.
*
* @param delSrc
* whether to delete the src
* @param src
* path
* @param dst
* path
* @param useRawLocalFileSystem
* whether to use RawLocalFileSystem as local file system or not.
*
* @throws IOException
* - if any IO error
*/
public void copyToLocalFile(boolean delSrc, Path src, Path dst,
boolean useRawLocalFileSystem) throws IOException {
Configuration conf = getConf();
FileSystem local = null;
if (useRawLocalFileSystem) {
local = getLocal(conf).getRawFileSystem();
} else {
local = getLocal(conf);
}
FileUtil.copy(this, src, local, dst, delSrc, conf);
}
/**

27
common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java
View File

@ -21,9 +21,11 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.After;
@ -1080,6 +1082,31 @@ public void testGetWrappedInputStream() throws IOException {
Assert.assertNotNull(is);
}
@Test
public void testCopyToLocalWithUseRawLocalFileSystemOption() throws Exception {
Configuration conf = new Configuration();
FileSystem fSys = new RawLocalFileSystem();
Path fileToFS = new Path(TEST_ROOT_DIR, "fs.txt");
Path fileToLFS = new Path(TEST_ROOT_DIR, "test.txt");
Path crcFileAtLFS = new Path(TEST_ROOT_DIR, ".test.txt.crc");
fSys.initialize(new URI("file:///"), conf);
writeFile(fSys, fileToFS);
if (fSys.exists(crcFileAtLFS))
Assert.assertTrue("CRC files not deleted", fSys
.delete(crcFileAtLFS, true));
fSys.copyToLocalFile(false, fileToFS, fileToLFS, true);
Assert.assertFalse("CRC files are created", fSys.exists(crcFileAtLFS));
}
private void writeFile(FileSystem fs, Path name) throws IOException {
FSDataOutputStream stm = fs.create(name);
try {
stm.writeBytes("42\n");
} finally {
stm.close();
}
}
protected void createFile(Path path) throws IOException {
FileSystemTestHelper.createFile(fSys, path);
}