big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1424566 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-12-20 16:19:48 +00:00
parent b324a6134e
commit 557b022406
4 changed files with 89 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -1224,6 +1224,8 @@ Release 0.23.6 - UNRELEASED
HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A.
Veselovsky via bobby)
HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby)
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
View File

@ -311,6 +311,7 @@ protected void processPaths(PathData parent, PathData ... items)
if (recursive && item.stat.isDirectory()) {
recursePath(item);
}
postProcessPath(item);
} catch (IOException e) {
displayError(e);
}
@ -329,6 +330,15 @@ protected void processPath(PathData item) throws IOException {
throw new RuntimeException("processPath() is not implemented");
}
/**
* Hook for commands to implement an operation to be applied on each
* path for the command after being processed successfully
* @param item a {@link PathData} object
* @throws IOException if anything goes wrong...
*/
protected void postProcessPath(PathData item) throws IOException {
}
/**
* Gets the directory listing for a path and invokes
* {@link #processPaths(PathData, PathData...)}

17
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.PathExistsException;
import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal;
/** Various commands for moving files */
@ -49,7 +50,21 @@ public static class MoveFromLocal extends CopyFromLocal {
@Override
protected void processPath(PathData src, PathData target) throws IOException {
target.fs.moveFromLocalFile(src.path, target.path);
// unlike copy, don't merge existing dirs during move
if (target.exists && target.stat.isDirectory()) {
throw new PathExistsException(target.toString());
}
super.processPath(src, target);
}
@Override
protected void postProcessPath(PathData src) throws IOException {
if (!src.fs.delete(src.path, false)) {
// we have no way to know the actual error...
PathIOException e = new PathIOException(src.toString());
e.setOperation("remove");
throw e;
}
}
}

60
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
View File

@ -358,6 +358,66 @@ public void testCopyMerge() throws Exception {
assertEquals("f1\ndf1\ndf2\ndf3\nf2\n", readFile("out"));
}
@Test
public void testMoveFileFromLocal() throws Exception {
Path testRoot = new Path(testRootDir, "testPutFile");
lfs.delete(testRoot, true);
lfs.mkdirs(testRoot);
Path target = new Path(testRoot, "target");
Path srcFile = new Path(testRoot, new Path("srcFile"));
lfs.createNewFile(srcFile);
int exit = shell.run(new String[]{
"-moveFromLocal", srcFile.toString(), target.toString() });
assertEquals(0, exit);
assertFalse(lfs.exists(srcFile));
assertTrue(lfs.exists(target));
assertTrue(lfs.isFile(target));
}
@Test
public void testMoveDirFromLocal() throws Exception {
Path testRoot = new Path(testRootDir, "testPutDir");
lfs.delete(testRoot, true);
lfs.mkdirs(testRoot);
Path srcDir = new Path(testRoot, "srcDir");
lfs.mkdirs(srcDir);
Path targetDir = new Path(testRoot, "target");
int exit = shell.run(new String[]{
"-moveFromLocal", srcDir.toString(), targetDir.toString() });
assertEquals(0, exit);
assertFalse(lfs.exists(srcDir));
assertTrue(lfs.exists(targetDir));
}
@Test
public void testMoveDirFromLocalDestExists() throws Exception {
Path testRoot = new Path(testRootDir, "testPutDir");
lfs.delete(testRoot, true);
lfs.mkdirs(testRoot);
Path srcDir = new Path(testRoot, "srcDir");
lfs.mkdirs(srcDir);
Path targetDir = new Path(testRoot, "target");
lfs.mkdirs(targetDir);
int exit = shell.run(new String[]{
"-moveFromLocal", srcDir.toString(), targetDir.toString() });
assertEquals(0, exit);
assertFalse(lfs.exists(srcDir));
assertTrue(lfs.exists(new Path(targetDir, srcDir.getName())));
lfs.mkdirs(srcDir);
exit = shell.run(new String[]{
"-moveFromLocal", srcDir.toString(), targetDir.toString() });
assertEquals(1, exit);
assertTrue(lfs.exists(srcDir));
}
private void createFile(Path ... paths) throws IOException {
for (Path path : paths) {
FSDataOutputStream out = lfs.create(path);