diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 67cd81ee91..397d81f92f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -214,6 +214,15 @@ public class CommonConfigurationKeysPublic { public static final String FS_TRASH_INTERVAL_KEY = "fs.trash.interval"; /** Default value for FS_TRASH_INTERVAL_KEY */ public static final long FS_TRASH_INTERVAL_DEFAULT = 0; + /** + * @see + * + * core-default.xml + */ + public static final String FS_TRASH_CLEAN_TRASHROOT_ENABLE_KEY = + "fs.trash.clean.trashroot.enable"; + /** Default value for FS_TRASH_CLEAN_TRASHROOT_ENABLE_KEY. */ + public static final boolean FS_TRASH_CLEAN_TRASHROOT_ENABLE_DEFAULT = false; /** * @see * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index f4228dea69..2fb4bff09f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -19,6 +19,8 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CLEAN_TRASHROOT_ENABLE_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CLEAN_TRASHROOT_ENABLE_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; @@ -70,6 +72,8 @@ public class TrashPolicyDefault extends TrashPolicy { private long emptierInterval; + private boolean cleanNonCheckpointUnderTrashRoot; + public TrashPolicyDefault() { } private TrashPolicyDefault(FileSystem fs, Configuration conf) @@ -90,6 +94,8 @@ public void initialize(Configuration conf, FileSystem fs, Path home) { this.emptierInterval = (long)(conf.getFloat( FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) * MSECS_PER_MINUTE); + this.cleanNonCheckpointUnderTrashRoot = conf.getBoolean( + FS_TRASH_CLEAN_TRASHROOT_ENABLE_KEY, FS_TRASH_CLEAN_TRASHROOT_ENABLE_DEFAULT); } @Override @@ -101,6 +107,8 @@ public void initialize(Configuration conf, FileSystem fs) { this.emptierInterval = (long)(conf.getFloat( FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) * MSECS_PER_MINUTE); + this.cleanNonCheckpointUnderTrashRoot = conf.getBoolean( + FS_TRASH_CLEAN_TRASHROOT_ENABLE_KEY, FS_TRASH_CLEAN_TRASHROOT_ENABLE_DEFAULT); if (deletionInterval < 0) { LOG.warn("Invalid value {} for deletion interval," + " deletion interaval can not be negative." @@ -374,8 +382,14 @@ private void deleteCheckpoint(Path trashRoot, boolean deleteImmediately) try { time = getTimeFromCheckpoint(name); } catch (ParseException e) { - LOG.warn("Unexpected item in trash: "+dir+". Ignoring."); - continue; + if (cleanNonCheckpointUnderTrashRoot) { + fs.delete(path, true); + LOG.warn("Unexpected item in trash: " + dir + ". Deleting."); + continue; + } else { + LOG.warn("Unexpected item in trash: " + dir + ". Ignoring."); + continue; + } } if (((now - deletionInterval) > time) || deleteImmediately) { diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index dd543deb8a..5f841bd233 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -974,6 +974,14 @@ + + fs.trash.clean.trashroot.enable + false + Whether clean some directories and files + in Trash home which are not under checkpoint directory. + + + fs.protected.directories diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java index 5b8c10b3fa..30c9a31fda 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java @@ -32,6 +32,7 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import org.junit.After; import org.junit.Before; @@ -786,6 +787,55 @@ public void testTrashEmptier() throws Exception { emptierThread.join(); } + /** + * Test trash emptier can delete non-checkpoint dir or not. + * @throws Exception + */ + @Test() + public void testTrashEmptierCleanDirNotInCheckpointDir() throws Exception { + Configuration conf = new Configuration(); + // Trash with 12 second deletes and 6 seconds checkpoints. + conf.set(FS_TRASH_INTERVAL_KEY, "0.2"); // 12 seconds + conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class); + conf.set(FS_TRASH_CHECKPOINT_INTERVAL_KEY, "0.1"); // 6 seconds + conf.setBoolean(FS_TRASH_CLEAN_TRASHROOT_ENABLE_KEY, true); + FileSystem fs = FileSystem.getLocal(conf); + conf.set("fs.default.name", fs.getUri().toString()); + + Trash trash = new Trash(conf); + + // Start Emptier in background. + Runnable emptier = trash.getEmptier(); + Thread emptierThread = new Thread(emptier); + emptierThread.start(); + + FsShell shell = new FsShell(); + shell.setConf(conf); + shell.init(); + + // Make sure the .Trash dir existed. + mkdir(fs, shell.getCurrentTrashDir()); + assertTrue(fs.exists(shell.getCurrentTrashDir())); + // Create a directory under .Trash directly. + Path myPath = new Path(shell.getCurrentTrashDir().getParent(), "test_dirs"); + mkdir(fs, myPath); + assertTrue(fs.exists(myPath)); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + try { + return !fs.exists(myPath); + } catch (IOException e) { + // Do nothing. + } + return false; + } + }, 6000, 60000); + emptierThread.interrupt(); + emptierThread.join(); + } + @After public void tearDown() throws IOException { File trashDir = new File(TEST_DIR.toUri().getPath());