HADOOP-6270. Support deleteOnExit in FileContext. Contributed by Suresh Srinivas

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@816751 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -571,6 +571,9 @@ Trunk (unreleased changes)
 
     HADOOP-6268. Add ivy jar to .gitignore. (Todd Lipcon via cdouglas)
 
+    HADOOP-6270. Support deleteOnExit in FileContext.  (Suresh Srinivas via
+    szetszwo)
+
   OPTIMIZATIONS
 
     HADOOP-5595. NameNode does not need to run a replicator to choose a

src/java/org/apache/hadoop/fs/FileContext.java

@@ -25,15 +25,23 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.Map.Entry;
 import java.util.regex.Pattern;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.classification.*;
-import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.*;
 
 /**
  * The FileContext class provides an interface to the application writer for
@@ -124,6 +132,18 @@
 
 public final class FileContext {
   
+  public static final Log LOG = LogFactory.getLog(FileContext.class);
+  
+  /**
+   * List of files that should be deleted on JVM shutdown
+   */
+  final static Map<FileContext, Set<Path>> deleteOnExit = 
+    new IdentityHashMap<FileContext, Set<Path>>();
+
+  /** JVM shutdown hook thread */
+  final static FileContextFinalizer finalizer = 
+    new FileContextFinalizer();
+  
   /**
    * The FileContext is defined by.
    *  1) defaultFS (slash)
@@ -179,6 +199,28 @@ private Path fixRelativePart(Path f) {
     }
   }
 
+  /**
+   * Delete all the paths that were marked as delete-on-exit.
+   */
+  static void processDeleteOnExit() {
+    synchronized (deleteOnExit) {
+      Set<Entry<FileContext, Set<Path>>> set = deleteOnExit.entrySet();
+      for (Entry<FileContext, Set<Path>> entry : set) {
+        FileContext fc = entry.getKey();
+        Set<Path> paths = entry.getValue();
+        for (Path path : paths) {
+          try {
+            fc.delete(path, true);
+          }
+          catch (IOException e) {
+            LOG.warn("Ignoring failure to deleteOnExit for path " + path);
+          }
+        }
+      }
+      deleteOnExit.clear();
+    }
+  }
+  
   /**
    * Pathnames with scheme and relative path are illegal.
    * @param path to be checked
@@ -886,6 +928,32 @@ public FileStatus[] listStatus(final Path f) throws IOException {
     return getFSofPath(absF).listStatus(absF);
   }
 
+  /**
+   * Mark a path to be deleted on JVM shutdown.
+   * 
+   * @param f the existing path to delete.
+   * @return  true if deleteOnExit is successful, otherwise false.
+   * @throws IOException
+   */
+  public boolean deleteOnExit(Path f) throws IOException {
+    if (!exists(f)) {
+      return false;
+    }
+    synchronized (deleteOnExit) {
+      if (deleteOnExit.isEmpty() && !finalizer.isAlive()) {
+        Runtime.getRuntime().addShutdownHook(finalizer);
+      }
+      
+      Set<Path> set = deleteOnExit.get(this);
+      if (set == null) {
+        set = new TreeSet<Path>();
+        deleteOnExit.put(this, set);
+      }
+      set.add(f);
+    }
+    return true;
+  }
+  
   private final Util util;
   public Util util() {
     return util;
@@ -1420,4 +1488,13 @@ private static boolean isSameFS(Path qualPath1, Path qualPath2) {
     return (srcUri.getAuthority().equals(dstUri.getAuthority()) && srcUri
         .getAuthority().equals(dstUri.getAuthority()));
   }
+  
+  /**
+   * Deletes all the paths in deleteOnExit on JVM shutdown
+   */
+  static class FileContextFinalizer extends Thread {
+    public synchronized void run() {
+      processDeleteOnExit();
+    }
+  }
 }

src/test/core/org/apache/hadoop/fs/TestFileContextDeleteOnExit.java

@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Set;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests {@link FileContext.#deleteOnExit(Path)} functionality.
+ */
+public class TestFileContextDeleteOnExit {
+  private static String TEST_ROOT_DIR =
+    System.getProperty("test.build.data", "/tmp") + "/test";
+  
+  private static byte[] data = new byte[1024 * 2]; // two blocks of data
+  {
+    for (int i = 0; i < data.length; i++) {
+      data[i] = (byte) (i % 10);
+    }
+  }
+  
+  private FileContext fc;
+  
+  @Before
+  public void setup() throws IOException {
+    fc = FileContext.getLocalFSFileContext();
+  }
+  
+  @After
+  public void tearDown() throws IOException {
+    fc.delete(getTestRootPath(), true);
+  }
+  
+  private Path getTestRootPath() {
+    return fc.makeQualified(new Path(TEST_ROOT_DIR));
+  }
+  
+  private Path getTestPath(String pathString) {
+    return fc.makeQualified(new Path(TEST_ROOT_DIR, pathString));
+  }
+  
+  private void createFile(FileContext fc, Path path) throws IOException {
+    FSDataOutputStream out = fc.create(path, EnumSet.of(CreateFlag.CREATE));
+    out.write(data, 0, data.length);
+    out.close();
+  }
+  
+  private void checkDeleteOnExitData(int size, FileContext fc, Path... paths) {
+    Assert.assertEquals(size, FileContext.deleteOnExit.size());
+    Set<Path> set = FileContext.deleteOnExit.get(fc);
+    Assert.assertEquals(paths.length, (set == null ? 0 : set.size()));
+    for (Path path : paths) {
+      Assert.assertTrue(set.contains(path));
+    }
+  }
+  
+  @Test
+  public void testDeleteOnExit() throws Exception {
+    // Create deleteOnExit entries
+    Path file1 = getTestPath("file1");
+    createFile(fc, file1);
+    fc.deleteOnExit(file1);
+    checkDeleteOnExitData(1, fc, file1);
+    
+    // Ensure shutdown hook is added
+    Assert.assertTrue(Runtime.getRuntime().removeShutdownHook(FileContext.finalizer));
+    
+    Path file2 = getTestPath("dir1/file2");
+    createFile(fc, file2);
+    fc.deleteOnExit(file2);
+    checkDeleteOnExitData(1, fc, file1, file2);
+    
+    Path dir = getTestPath("dir3/dir4/dir5/dir6");
+    createFile(fc, dir);
+    fc.deleteOnExit(dir);
+    checkDeleteOnExitData(1, fc, file1, file2, dir);
+    
+    // trigger deleteOnExit and ensure the registered
+    // paths are cleaned up
+    FileContext.finalizer.start();
+    FileContext.finalizer.join();
+    checkDeleteOnExitData(0, fc, new Path[0]);
+    Assert.assertFalse(fc.exists(file1));
+    Assert.assertFalse(fc.exists(file2));
+    Assert.assertFalse(fc.exists(dir));
+  }
+}