diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4a47c4fa62..3b9009d40b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1224,6 +1224,8 @@ Release 0.23.6 - UNRELEASED HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A. Veselovsky via bobby) + HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java index f229d1f223..8c5e88058e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java @@ -311,6 +311,7 @@ protected void processPaths(PathData parent, PathData ... items) if (recursive && item.stat.isDirectory()) { recursePath(item); } + postProcessPath(item); } catch (IOException e) { displayError(e); } @@ -329,6 +330,15 @@ protected void processPath(PathData item) throws IOException { throw new RuntimeException("processPath() is not implemented"); } + /** + * Hook for commands to implement an operation to be applied on each + * path for the command after being processed successfully + * @param item a {@link PathData} object + * @throws IOException if anything goes wrong... + */ + protected void postProcessPath(PathData item) throws IOException { + } + /** * Gets the directory listing for a path and invokes * {@link #processPaths(PathData, PathData...)} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java index e1ad1a2ef6..ffb3483ef2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.PathExistsException; import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal; /** Various commands for moving files */ @@ -49,7 +50,21 @@ public static class MoveFromLocal extends CopyFromLocal { @Override protected void processPath(PathData src, PathData target) throws IOException { - target.fs.moveFromLocalFile(src.path, target.path); + // unlike copy, don't merge existing dirs during move + if (target.exists && target.stat.isDirectory()) { + throw new PathExistsException(target.toString()); + } + super.processPath(src, target); + } + + @Override + protected void postProcessPath(PathData src) throws IOException { + if (!src.fs.delete(src.path, false)) { + // we have no way to know the actual error... + PathIOException e = new PathIOException(src.toString()); + e.setOperation("remove"); + throw e; + } } } @@ -95,4 +110,4 @@ protected void processPath(PathData src, PathData target) throws IOException { } } } -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java index 5b4f585c19..72df11e6df 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java @@ -357,6 +357,66 @@ public void testCopyMerge() throws Exception { assertEquals(0, exit); assertEquals("f1\ndf1\ndf2\ndf3\nf2\n", readFile("out")); } + + + @Test + public void testMoveFileFromLocal() throws Exception { + Path testRoot = new Path(testRootDir, "testPutFile"); + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + + Path target = new Path(testRoot, "target"); + Path srcFile = new Path(testRoot, new Path("srcFile")); + lfs.createNewFile(srcFile); + + int exit = shell.run(new String[]{ + "-moveFromLocal", srcFile.toString(), target.toString() }); + assertEquals(0, exit); + assertFalse(lfs.exists(srcFile)); + assertTrue(lfs.exists(target)); + assertTrue(lfs.isFile(target)); + } + + @Test + public void testMoveDirFromLocal() throws Exception { + Path testRoot = new Path(testRootDir, "testPutDir"); + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + + Path srcDir = new Path(testRoot, "srcDir"); + lfs.mkdirs(srcDir); + Path targetDir = new Path(testRoot, "target"); + + int exit = shell.run(new String[]{ + "-moveFromLocal", srcDir.toString(), targetDir.toString() }); + assertEquals(0, exit); + assertFalse(lfs.exists(srcDir)); + assertTrue(lfs.exists(targetDir)); + } + + @Test + public void testMoveDirFromLocalDestExists() throws Exception { + Path testRoot = new Path(testRootDir, "testPutDir"); + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + + Path srcDir = new Path(testRoot, "srcDir"); + lfs.mkdirs(srcDir); + Path targetDir = new Path(testRoot, "target"); + lfs.mkdirs(targetDir); + + int exit = shell.run(new String[]{ + "-moveFromLocal", srcDir.toString(), targetDir.toString() }); + assertEquals(0, exit); + assertFalse(lfs.exists(srcDir)); + assertTrue(lfs.exists(new Path(targetDir, srcDir.getName()))); + + lfs.mkdirs(srcDir); + exit = shell.run(new String[]{ + "-moveFromLocal", srcDir.toString(), targetDir.toString() }); + assertEquals(1, exit); + assertTrue(lfs.exists(srcDir)); + } private void createFile(Path ... paths) throws IOException { for (Path path : paths) {