big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-10030. FsShell -put/copyFromLocal should support Windows local path. Contributed by Chuan Liu.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1530462 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2013-10-08 23:41:53 +00:00
parent 5c02d2f622
commit b7e885435a
3 changed files with 66 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -407,6 +407,9 @@ Release 2.2.1 - UNRELEASED
HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9)
HADOOP-10030. FsShell -put/copyFromLocal should support Windows local path.
(Chuan Liu via cnauroth)
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES

11
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
View File

@ -204,13 +204,18 @@ protected void processOptions(LinkedList<String> args) throws IOException {
// commands operating on local paths have no need for glob expansion
@Override
protected List<PathData> expandArgument(String arg) throws IOException {
List<PathData> items = new LinkedList<PathData>();
try {
List<PathData> items = new LinkedList<PathData>();
items.add(new PathData(new URI(arg), getConf()));
return items;
} catch (URISyntaxException e) {
throw new IOException("unexpected URISyntaxException", e);
if (Path.WINDOWS) {
// Unlike URI, PathData knows how to parse Windows drive-letter paths.
items.add(new PathData(arg, getConf()));
} else {
throw new IOException("unexpected URISyntaxException", e);
}
}
return items;
}
@Override

71
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
View File

@ -19,7 +19,9 @@
package org.apache.hadoop.fs;
import static org.junit.Assert.*;
import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
@ -106,7 +108,7 @@ public void testCopyFileFromLocal() throws Exception {
Path targetDir = new Path(testRoot, "target");
Path filePath = new Path(testRoot, new Path("srcFile"));
lfs.create(filePath).close();
checkPut(filePath, targetDir);
checkPut(filePath, targetDir, false);
}
@Test
@ -119,10 +121,42 @@ public void testCopyDirFromLocal() throws Exception {
Path dirPath = new Path(testRoot, new Path("srcDir"));
lfs.mkdirs(dirPath);
lfs.create(new Path(dirPath, "srcFile")).close();
checkPut(dirPath, targetDir);
checkPut(dirPath, targetDir, false);
}
@Test
public void testCopyFileFromWindowsLocalPath() throws Exception {
assumeTrue(Path.WINDOWS);
String windowsTestRootPath = (new File(testRootDir.toUri().getPath()
.toString())).getAbsolutePath();
Path testRoot = new Path(windowsTestRootPath, "testPutFile");
lfs.delete(testRoot, true);
lfs.mkdirs(testRoot);
Path targetDir = new Path(testRoot, "target");
Path filePath = new Path(testRoot, new Path("srcFile"));
lfs.create(filePath).close();
checkPut(filePath, targetDir, true);
}
@Test
public void testCopyDirFromWindowsLocalPath() throws Exception {
assumeTrue(Path.WINDOWS);
String windowsTestRootPath = (new File(testRootDir.toUri().getPath()
.toString())).getAbsolutePath();
Path testRoot = new Path(windowsTestRootPath, "testPutDir");
lfs.delete(testRoot, true);
lfs.mkdirs(testRoot);
Path targetDir = new Path(testRoot, "target");
Path dirPath = new Path(testRoot, new Path("srcDir"));
lfs.mkdirs(dirPath);
lfs.create(new Path(dirPath, "srcFile")).close();
checkPut(dirPath, targetDir, true);
}
private void checkPut(Path srcPath, Path targetDir)
private void checkPut(Path srcPath, Path targetDir, boolean useWindowsPath)
throws Exception {
lfs.delete(targetDir, true);
lfs.mkdirs(targetDir);
@ -134,37 +168,37 @@ private void checkPut(Path srcPath, Path targetDir)
// copy to new file, then again
prepPut(dstPath, false, false);
checkPut(0, srcPath, dstPath);
checkPut(0, srcPath, dstPath, useWindowsPath);
if (lfs.isFile(srcPath)) {
checkPut(1, srcPath, dstPath);
checkPut(1, srcPath, dstPath, useWindowsPath);
} else { // directory works because it copies into the dir
// clear contents so the check won't think there are extra paths
prepPut(dstPath, true, true);
checkPut(0, srcPath, dstPath);
checkPut(0, srcPath, dstPath, useWindowsPath);
}
// copy to non-existent subdir
prepPut(childPath, false, false);
checkPut(1, srcPath, dstPath);
checkPut(1, srcPath, dstPath, useWindowsPath);
// copy into dir, then with another name
prepPut(dstPath, true, true);
checkPut(0, srcPath, dstPath);
checkPut(0, srcPath, dstPath, useWindowsPath);
prepPut(childPath, true, true);
checkPut(0, srcPath, childPath);
checkPut(0, srcPath, childPath, useWindowsPath);
// try to put to pwd with existing dir
prepPut(targetDir, true, true);
checkPut(0, srcPath, null);
checkPut(0, srcPath, null, useWindowsPath);
prepPut(targetDir, true, true);
checkPut(0, srcPath, new Path("."));
checkPut(0, srcPath, new Path("."), useWindowsPath);
// try to put to pwd with non-existent cwd
prepPut(dstPath, false, true);
lfs.setWorkingDirectory(dstPath);
checkPut(1, srcPath, null);
checkPut(1, srcPath, null, useWindowsPath);
prepPut(dstPath, false, true);
checkPut(1, srcPath, new Path("."));
checkPut(1, srcPath, new Path("."), useWindowsPath);
}
private void prepPut(Path dst, boolean create,
@ -183,12 +217,17 @@ private void prepPut(Path dst, boolean create,
}
}
private void checkPut(int exitCode, Path src, Path dest) throws Exception {
private void checkPut(int exitCode, Path src, Path dest,
boolean useWindowsPath) throws Exception {
String argv[] = null;
String srcPath = src.toString();
if (useWindowsPath) {
srcPath = (new File(srcPath)).getAbsolutePath();
}
if (dest != null) {
argv = new String[]{ "-put", src.toString(), pathAsString(dest) };
argv = new String[]{ "-put", srcPath, pathAsString(dest) };
} else {
argv = new String[]{ "-put", src.toString() };
argv = new String[]{ "-put", srcPath };
dest = new Path(Path.CUR_DIR);
}