From 7226dead1e66e501077fe1a0ec04369f03ecd2d2 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 30 Jul 2012 20:58:44 +0000 Subject: [PATCH] HADOOP-8635. Cannot cancel paths registered deleteOnExit (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367296 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../java/org/apache/hadoop/fs/FileSystem.java | 10 +++ .../hadoop/fs/TestFileSystemCaching.java | 88 +++++++++++++++---- .../hadoop/fs/TestFilterFileSystem.java | 3 + 4 files changed, 87 insertions(+), 16 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d1faef370a..03e41de3da 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -865,6 +865,8 @@ Release 0.23.3 - UNRELEASED HADOOP-8550. hadoop fs -touchz automatically created parent directories (John George via bobby) + HADOOP-8635. Cannot cancel paths registered deleteOnExit (daryn via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 42b30c4999..2fa4bd0c03 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -1214,6 +1214,16 @@ public abstract class FileSystem extends Configured implements Closeable { } return true; } + + /** + * Cancel the deletion of the path when the FileSystem is closed + * @param f the path to cancel deletion + */ + public boolean cancelDeleteOnExit(Path f) { + synchronized (deleteOnExit) { + return deleteOnExit.remove(f); + } + } /** * Delete all files that were marked as delete-on-exit. This recursively diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java index e8debf8dfa..7e5f99f5fb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java @@ -269,26 +269,82 @@ public class TestFileSystemCaching { } @Test - public void testDeleteOnExitChecksExists() throws Exception { + public void testDelete() throws IOException { + FileSystem mockFs = mock(FileSystem.class); + FileSystem fs = new FilterFileSystem(mockFs); + Path path = new Path("/a"); + + fs.delete(path, false); + verify(mockFs).delete(eq(path), eq(false)); + reset(mockFs); + fs.delete(path, true); + verify(mockFs).delete(eq(path), eq(true)); + } + + @Test + public void testDeleteOnExit() throws IOException { FileSystem mockFs = mock(FileSystem.class); FileSystem fs = new FilterFileSystem(mockFs); - Path p = new Path("/a"); - - // path has to exist for deleteOnExit to register it - when(mockFs.getFileStatus(p)).thenReturn(new FileStatus()); - fs.deleteOnExit(p); - verify(mockFs).getFileStatus(eq(p)); - fs.close(); - verify(mockFs).delete(eq(p), anyBoolean()); + Path path = new Path("/a"); + + // delete on close if path does exist + when(mockFs.getFileStatus(eq(path))).thenReturn(new FileStatus()); + assertTrue(fs.deleteOnExit(path)); + verify(mockFs).getFileStatus(eq(path)); reset(mockFs); - - // make sure it doesn't try to delete a file that doesn't exist - when(mockFs.getFileStatus(p)).thenReturn(new FileStatus()); - fs.deleteOnExit(p); - verify(mockFs).getFileStatus(eq(p)); + when(mockFs.getFileStatus(eq(path))).thenReturn(new FileStatus()); + fs.close(); + verify(mockFs).getFileStatus(eq(path)); + verify(mockFs).delete(eq(path), eq(true)); + } + + @Test + public void testDeleteOnExitFNF() throws IOException { + FileSystem mockFs = mock(FileSystem.class); + FileSystem fs = new FilterFileSystem(mockFs); + Path path = new Path("/a"); + + // don't delete on close if path doesn't exist + assertFalse(fs.deleteOnExit(path)); + verify(mockFs).getFileStatus(eq(path)); reset(mockFs); fs.close(); - verify(mockFs).getFileStatus(eq(p)); + verify(mockFs, never()).getFileStatus(eq(path)); verify(mockFs, never()).delete(any(Path.class), anyBoolean()); } -} \ No newline at end of file + + + @Test + public void testDeleteOnExitRemoved() throws IOException { + FileSystem mockFs = mock(FileSystem.class); + FileSystem fs = new FilterFileSystem(mockFs); + Path path = new Path("/a"); + + // don't delete on close if path existed, but later removed + when(mockFs.getFileStatus(eq(path))).thenReturn(new FileStatus()); + assertTrue(fs.deleteOnExit(path)); + verify(mockFs).getFileStatus(eq(path)); + reset(mockFs); + fs.close(); + verify(mockFs).getFileStatus(eq(path)); + verify(mockFs, never()).delete(any(Path.class), anyBoolean()); + } + + @Test + public void testCancelDeleteOnExit() throws IOException { + FileSystem mockFs = mock(FileSystem.class); + FileSystem fs = new FilterFileSystem(mockFs); + Path path = new Path("/a"); + + // don't delete on close if path existed, but later cancelled + when(mockFs.getFileStatus(eq(path))).thenReturn(new FileStatus()); + assertTrue(fs.deleteOnExit(path)); + verify(mockFs).getFileStatus(eq(path)); + assertTrue(fs.cancelDeleteOnExit(path)); + assertFalse(fs.cancelDeleteOnExit(path)); // false because not registered + reset(mockFs); + fs.close(); + verify(mockFs, never()).getFileStatus(any(Path.class)); + verify(mockFs, never()).delete(any(Path.class), anyBoolean()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java index a374c08a18..727986dcd1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java @@ -182,6 +182,9 @@ public class TestFilterFileSystem { public boolean deleteOnExit(Path f) throws IOException { return false; } + public boolean cancelDeleteOnExit(Path f) throws IOException { + return false; + } public String getScheme() { return "dontcheck"; }