HDFS-3094. add -nonInteractive and -force option to namenode -format command. Contributed by Arpit Gupta.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1312025 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -355,6 +355,9 @@ Release 2.0.0 - UNRELEASED
 
     HDFS-3238. ServerCommand and friends don't need to be writables. (eli)
 
+    HDFS-3094. add -nonInteractive and -force option to namenode -format
+    command (Arpit Gupta via todd)
+
   OPTIMIZATIONS
 
     HDFS-3024. Improve performance of stringification in addStoredBlock (todd)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java

@@ -57,13 +57,18 @@ static public enum StartupOption{
     BOOTSTRAPSTANDBY("-bootstrapStandby"),
     INITIALIZESHAREDEDITS("-initializeSharedEdits"),
     RECOVER  ("-recover"),
-    FORCE("-force");
+    FORCE("-force"),
+    NONINTERACTIVE("-nonInteractive");
     
     private String name = null;
     
     // Used only with format and upgrade options
     private String clusterId = null;
     
+    // Used only with format option
+    private boolean isForceFormat = false;
+    private boolean isInteractiveFormat = true;
+    
     // Used only with recovery option
     private int force = 0;
 
@@ -101,6 +106,22 @@ public void setForce(int force) {
     public int getForce() {
       return this.force;
     }
+    
+    public boolean getForceFormat() {
+      return isForceFormat;
+    }
+    
+    public void setForceFormat(boolean force) {
+      isForceFormat = force;
+    }
+    
+    public boolean getInteractiveFormat() {
+      return isInteractiveFormat;
+    }
+    
+    public void setInteractiveFormat(boolean interactive) {
+      isInteractiveFormat = interactive;
+    }
   }
 
   // Timeouts for communicating with DataNode for streaming writes/reads

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -215,7 +215,7 @@ public long getProtocolVersion(String protocol,
   /** Format a new filesystem.  Destroys any filesystem that may already
    * exist at this location.  **/
   public static void format(Configuration conf) throws IOException {
-    format(conf, true);
+    format(conf, true, true);
   }
 
   static NameNodeMetrics metrics;
@@ -658,9 +658,8 @@ public InetSocketAddress getHttpAddress() {
    * @return true if formatting was aborted, false otherwise
    * @throws IOException
    */
-  private static boolean format(Configuration conf,
-                                boolean force)
-      throws IOException {
+  private static boolean format(Configuration conf, boolean force,
+      boolean isInteractive) throws IOException {
     String nsId = DFSUtil.getNamenodeNameServiceId(conf);
     String namenodeId = HAUtil.getNameNodeId(conf, nsId);
     initializeGenericKeys(conf, nsId, namenodeId);
@@ -669,7 +668,7 @@ private static boolean format(Configuration conf,
     Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
     List<URI> editDirsToFormat = 
                  FSNamesystem.getNamespaceEditsDirs(conf);
-    if (!confirmFormat(dirsToFormat, force, true)) {
+    if (!confirmFormat(dirsToFormat, force, isInteractive)) {
       return true; // aborted
     }
 
@@ -830,8 +829,9 @@ private static void printUsage() {
       "Usage: java NameNode [" +
       StartupOption.BACKUP.getName() + "] | [" +
       StartupOption.CHECKPOINT.getName() + "] | [" +
-      StartupOption.FORMAT.getName() + "[" + StartupOption.CLUSTERID.getName() +  
-      " cid ]] | [" +
+      StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() +  
+      " cid ] [" + StartupOption.FORCE.getName() + "] [" +
+      StartupOption.NONINTERACTIVE.getName() + "] ] | [" +
       StartupOption.UPGRADE.getName() + "] | [" +
       StartupOption.ROLLBACK.getName() + "] | [" +
       StartupOption.FINALIZE.getName() + "] | [" +
@@ -850,11 +850,35 @@ private static StartupOption parseArguments(String args[]) {
       String cmd = args[i];
       if (StartupOption.FORMAT.getName().equalsIgnoreCase(cmd)) {
         startOpt = StartupOption.FORMAT;
-        // might be followed by two args
-        if (i + 2 < argsLen
-            && args[i + 1].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
-          i += 2;
-          startOpt.setClusterId(args[i]);
+        for (i = i + 1; i < argsLen; i++) {
+          if (args[i].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
+            i++;
+            if (i >= argsLen) {
+              // if no cluster id specified, return null
+              LOG.fatal("Must specify a valid cluster ID after the "
+                  + StartupOption.CLUSTERID.getName() + " flag");
+              return null;
+            }
+            String clusterId = args[i];
+            // Make sure an id is specified and not another flag
+            if (clusterId.isEmpty() ||
+                clusterId.equalsIgnoreCase(StartupOption.FORCE.getName()) ||
+                clusterId.equalsIgnoreCase(
+                    StartupOption.NONINTERACTIVE.getName())) {
+              LOG.fatal("Must specify a valid cluster ID after the "
+                  + StartupOption.CLUSTERID.getName() + " flag");
+              return null;
+            }
+            startOpt.setClusterId(clusterId);
+          }
+
+          if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) {
+            startOpt.setForceFormat(true);
+          }
+
+          if (args[i].equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) {
+            startOpt.setInteractiveFormat(false);
+          }
         }
       } else if (StartupOption.GENCLUSTERID.getName().equalsIgnoreCase(cmd)) {
         startOpt = StartupOption.GENCLUSTERID;
@@ -997,7 +1021,8 @@ public static NameNode createNameNode(String argv[], Configuration conf)
 
     switch (startOpt) {
       case FORMAT: {
-        boolean aborted = format(conf, false);
+        boolean aborted = format(conf, startOpt.getForceFormat(),
+            startOpt.getInteractiveFormat());
         System.exit(aborted ? 1 : 0);
         return null; // avoid javac warning
       }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java

@@ -18,12 +18,19 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
 import java.net.URI;
-import java.util.ArrayList;
+import java.security.Permission;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -40,11 +47,11 @@
 import org.junit.Before;
 import org.junit.Test;
 
-
 public class TestClusterId {
   private static final Log LOG = LogFactory.getLog(TestClusterId.class);
   File hdfsDir;
-  
+  Configuration config;
+
   private String getClusterId(Configuration config) throws IOException {
     // see if cluster id not empty.
     Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(config);
@@ -59,33 +66,41 @@ private String getClusterId(Configuration config) throws IOException {
     LOG.info("successfully formated : sd="+sd.getCurrentDir() + ";cid="+cid);
     return cid;
   }
-  
+
   @Before
   public void setUp() throws IOException {
+    System.setSecurityManager(new NoExitSecurityManager());
+
     String baseDir = System.getProperty("test.build.data", "build/test/data");
 
-    hdfsDir = new File(baseDir, "dfs");
-    if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
-      throw new IOException("Could not delete test directory '" + 
-          hdfsDir + "'");
+    hdfsDir = new File(baseDir, "dfs/name");
+    if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
+      throw new IOException("Could not delete test directory '" + hdfsDir + "'");
     }
     LOG.info("hdfsdir is " + hdfsDir.getAbsolutePath());
+
+    // as some tests might change these values we reset them to defaults before
+    // every test
+    StartupOption.FORMAT.setForceFormat(false);
+    StartupOption.FORMAT.setInteractiveFormat(true);
+    
+    config = new Configuration();
+    config.set(DFS_NAMENODE_NAME_DIR_KEY, hdfsDir.getPath());
   }
-  
+
   @After
   public void tearDown() throws IOException {
-    if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
-      throw new IOException("Could not tearDown test directory '" +
-          hdfsDir + "'");
+    System.setSecurityManager(null);
+
+    if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
+      throw new IOException("Could not tearDown test directory '" + hdfsDir
+          + "'");
     }
   }
-  
+
   @Test
   public void testFormatClusterIdOption() throws IOException {
-    Configuration config = new Configuration();
     
-    config.set(DFS_NAMENODE_NAME_DIR_KEY, new File(hdfsDir, "name").getPath());
-
     // 1. should format without cluster id
     //StartupOption.FORMAT.setClusterId("");
     NameNode.format(config);
@@ -107,4 +122,356 @@ public void testFormatClusterIdOption() throws IOException {
     String newCid = getClusterId(config);
     assertFalse("ClusterId should not be the same", newCid.equals(cid));
   }
-}
+
+  /**
+   * Test namenode format with -format option. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormat() throws IOException {
+    String[] argv = { "-format" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format option when an empty name directory
+   * exists. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithEmptyDir() throws IOException {
+
+    if (!hdfsDir.mkdirs()) {
+      fail("Failed to create dir " + hdfsDir.getPath());
+    }
+
+    String[] argv = { "-format" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format -force options when name directory
+   * exists. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithForce() throws IOException {
+
+    if (!hdfsDir.mkdirs()) {
+      fail("Failed to create dir " + hdfsDir.getPath());
+    }
+
+    String[] argv = { "-format", "-force" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format -force -clusterid option when name
+   * directory exists. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithForceAndClusterId() throws IOException {
+
+    if (!hdfsDir.mkdirs()) {
+      fail("Failed to create dir " + hdfsDir.getPath());
+    }
+
+    String myId = "testFormatWithForceAndClusterId";
+    String[] argv = { "-format", "-force", "-clusterid", myId };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cId = getClusterId(config);
+    assertEquals("ClusterIds do not match", myId, cId);
+  }
+
+  /**
+   * Test namenode format with -clusterid -force option. Format command should
+   * fail as no cluster id was provided.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithInvalidClusterIdOption() throws IOException {
+
+    String[] argv = { "-format", "-clusterid", "-force" };
+    PrintStream origErr = System.err;
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintStream stdErr = new PrintStream(baos);
+    System.setErr(stdErr);
+
+    NameNode.createNameNode(argv, config);
+
+    // Check if usage is printed
+    assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
+    System.setErr(origErr);
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  /**
+   * Test namenode format with -format -clusterid options. Format should fail
+   * was no clusterid was sent.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithNoClusterIdOption() throws IOException {
+
+    String[] argv = { "-format", "-clusterid" };
+    PrintStream origErr = System.err;
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintStream stdErr = new PrintStream(baos);
+    System.setErr(stdErr);
+
+    NameNode.createNameNode(argv, config);
+
+    // Check if usage is printed
+    assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
+    System.setErr(origErr);
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  /**
+   * Test namenode format with -format -clusterid and empty clusterid. Format
+   * should fail as no valid if was provided.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithEmptyClusterIdOption() throws IOException {
+
+    String[] argv = { "-format", "-clusterid", "" };
+
+    PrintStream origErr = System.err;
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintStream stdErr = new PrintStream(baos);
+    System.setErr(stdErr);
+
+    NameNode.createNameNode(argv, config);
+
+    // Check if usage is printed
+    assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
+    System.setErr(origErr);
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  /**
+   * Test namenode format with -format -nonInteractive options when a non empty
+   * name directory exists. Format should not succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithNonInteractive() throws IOException {
+
+    // we check for a non empty dir, so create a child path
+    File data = new File(hdfsDir, "file");
+    if (!data.mkdirs()) {
+      fail("Failed to create dir " + data.getPath());
+    }
+
+    String[] argv = { "-format", "-nonInteractive" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have been aborted with exit code 1", 1,
+          e.status);
+    }
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  /**
+   * Test namenode format with -format -nonInteractive options when name
+   * directory does not exist. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithNonInteractiveNameDirDoesNotExit()
+      throws IOException {
+
+    String[] argv = { "-format", "-nonInteractive" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -force -nonInteractive -force option. Format
+   * should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithNonInteractiveAndForce() throws IOException {
+
+    if (!hdfsDir.mkdirs()) {
+      fail("Failed to create dir " + hdfsDir.getPath());
+    }
+
+    String[] argv = { "-format", "-nonInteractive", "-force" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format option when a non empty name directory
+   * exists. Enter Y when prompted and the format should succeed.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testFormatWithoutForceEnterYes() throws IOException,
+      InterruptedException {
+
+    // we check for a non empty dir, so create a child path
+    File data = new File(hdfsDir, "file");
+    if (!data.mkdirs()) {
+      fail("Failed to create dir " + data.getPath());
+    }
+
+    // capture the input stream
+    InputStream origIn = System.in;
+    ByteArrayInputStream bins = new ByteArrayInputStream("Y\n".getBytes());
+    System.setIn(bins);
+
+    String[] argv = { "-format" };
+
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    System.setIn(origIn);
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format option when a non empty name directory
+   * exists. Enter N when prompted and format should be aborted.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testFormatWithoutForceEnterNo() throws IOException,
+      InterruptedException {
+
+    // we check for a non empty dir, so create a child path
+    File data = new File(hdfsDir, "file");
+    if (!data.mkdirs()) {
+      fail("Failed to create dir " + data.getPath());
+    }
+
+    // capture the input stream
+    InputStream origIn = System.in;
+    ByteArrayInputStream bins = new ByteArrayInputStream("N\n".getBytes());
+    System.setIn(bins);
+
+    String[] argv = { "-format" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should not have succeeded", 1, e.status);
+    }
+
+    System.setIn(origIn);
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  private static class ExitException extends SecurityException {
+    private static final long serialVersionUID = 1L;
+    public final int status;
+
+    public ExitException(int status) {
+      super("There is no escape!");
+      this.status = status;
+    }
+  }
+
+  private static class NoExitSecurityManager extends SecurityManager {
+    @Override
+    public void checkPermission(Permission perm) {
+      // allow anything.
+    }
+
+    @Override
+    public void checkPermission(Permission perm, Object context) {
+      // allow anything.
+    }
+
+    @Override
+    public void checkExit(int status) {
+      super.checkExit(status);
+      throw new ExitException(status);
+    }
+  }
+}