HDFS-2188. Make FSEditLog create its journals from a list of URIs rather than NNStorage. Contributed by Ivan Kelly.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1185354 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -129,6 +129,9 @@ Trunk (unreleased changes)
     file or creating a file without specifying the replication parameter.
     (szetszwo)
 
+    HDFS-2188. Make FSEditLog create its journals from a list of URIs rather 
+    than NNStorage. (Ivan Kelly via jitendra)
+
 Release 0.23.0 - Unreleased
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java

@@ -18,10 +18,11 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import static org.apache.hadoop.hdfs.server.common.Util.now;
-
+import java.net.URI;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -42,9 +43,11 @@
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.conf.Configuration;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 
 /**
  * FSEditLog maintains a log of the namespace modifications.
@@ -122,23 +125,68 @@ protected synchronized TransactionId initialValue() {
     }
   };
 
+  final private Collection<URI> editsDirs;
+
+  /**
+   * Construct FSEditLog with default configuration, taking editDirs from NNStorage
+   * @param storage Storage object used by namenode
+   */
+  @VisibleForTesting
   FSEditLog(NNStorage storage) {
+    this(new Configuration(), storage, Collections.<URI>emptyList());
+  }
+
+  /**
+   * Constructor for FSEditLog. Add underlying journals are constructed, but 
+   * no streams are opened until open() is called.
+   * 
+   * @param conf The namenode configuration
+   * @param storage Storage object used by namenode
+   * @param editsDirs List of journals to use
+   */
+  FSEditLog(Configuration conf, NNStorage storage, Collection<URI> editsDirs) {
     isSyncRunning = false;
     this.storage = storage;
     metrics = NameNode.getNameNodeMetrics();
     lastPrintTime = now();
+    
+    if (editsDirs.isEmpty()) { 
+      // if this is the case, no edit dirs have been explictly configured
+      // image dirs are to be used for edits too
+      try {
+        editsDirs = Lists.newArrayList(storage.getEditsDirectories());
+      } catch (IOException ioe) {
+        // cannot get list from storage, so the empty editsDirs 
+        // will be assigned. an error will be thrown on first use
+        // of the editlog, as no journals will exist
+      }
+      this.editsDirs = editsDirs;
+    } else {
+      this.editsDirs = Lists.newArrayList(editsDirs);
+    }
 
     this.journalSet = new JournalSet();
-    for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
+    for (URI u : this.editsDirs) {
-      journalSet.add(new FileJournalManager(sd));
+      StorageDirectory sd = storage.getStorageDirectory(u);
+      if (sd != null) {
+        journalSet.add(new FileJournalManager(sd));
+      }
     }
-    
+ 
     if (journalSet.isEmpty()) {
       LOG.error("No edits directories configured!");
     } 
     state = State.BETWEEN_LOG_SEGMENTS;
   }
-  
+
+  /**
+   * Get the list of URIs the editlog is using for storage
+   * @return collection of URIs in use by the edit log
+   */
+  Collection<URI> getEditURIs() {
+    return editsDirs;
+  }
+
   /**
    * Initialize the output stream for logging, opening the first
    * log segment.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java

@@ -120,7 +120,7 @@ protected FSImage(Configuration conf,
       storage.setRestoreFailedStorage(true);
     }
 
-    this.editLog = new FSEditLog(storage);
+    this.editLog = new FSEditLog(conf, storage, editsDirs);
     
     archivalManager = new NNStorageRetentionManager(conf, storage, editLog);
   }
@@ -150,8 +150,7 @@ boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target)
       "NameNode formatting should be performed before reading the image";
     
     Collection<URI> imageDirs = storage.getImageDirectories();
-    Collection<URI> editsDirs = storage.getEditsDirectories();
+    Collection<URI> editsDirs = editLog.getEditURIs();
-
 
     // none of the data dirs exist
     if((imageDirs.size() == 0 || editsDirs.size() == 0) 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java

@@ -59,6 +59,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
 
 /**
  * NNStorage is responsible for management of the StorageDirectories used by
@@ -154,7 +155,9 @@ public NNStorage(Configuration conf,
 
     storageDirs = new CopyOnWriteArrayList<StorageDirectory>();
     
-    setStorageDirectories(imageDirs, editsDirs);
+    // this may modify the editsDirs, so copy before passing in
+    setStorageDirectories(imageDirs, 
+                          Lists.newArrayList(editsDirs));
   }
 
   @Override // Storage
@@ -298,6 +301,27 @@ synchronized void setStorageDirectories(Collection<URI> fsNameDirs,
     }
   }
 
+  /**
+   * Return the storage directory corresponding to the passed URI
+   * @param uri URI of a storage directory
+   * @return The matching storage directory or null if none found
+   */
+  StorageDirectory getStorageDirectory(URI uri) {
+    try {
+      uri = Util.fileAsURI(new File(uri));
+      Iterator<StorageDirectory> it = dirIterator();
+      for (; it.hasNext(); ) {
+        StorageDirectory sd = it.next();
+        if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
+          return sd;
+        }
+      }
+    } catch (IOException ioe) {
+      LOG.warn("Error converting file to URI", ioe);
+    }
+    return null;
+  }
+
   /**
    * Checks the consistency of a URI, in particular if the scheme
    * is specified and is supported by a concrete implementation

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java

@@ -42,10 +42,13 @@
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
 import org.mockito.Mockito;
+import org.mockito.Matchers;
 
 import com.google.common.base.Joiner;
 import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.io.Files;
@@ -106,7 +109,7 @@ public static StorageDirectory mockStorageDirectory(
     Mockito.doReturn(type)
       .when(sd).getStorageDirType();
     Mockito.doReturn(currentDir).when(sd).getCurrentDir();
-    
+    Mockito.doReturn(currentDir).when(sd).getRoot();
     Mockito.doReturn(mockFile(true)).when(sd).getVersionFile();
     Mockito.doReturn(mockFile(false)).when(sd).getPreviousDir();
     return sd;
@@ -128,7 +131,8 @@ static StorageDirectory mockStorageDirectory(
   
     // Version file should always exist
     doReturn(mockFile(true)).when(sd).getVersionFile();
-    
+    doReturn(mockFile(true)).when(sd).getRoot();
+
     // Previous dir optionally exists
     doReturn(mockFile(previousExists))
       .when(sd).getPreviousDir();   
@@ -143,6 +147,7 @@ static StorageDirectory mockStorageDirectory(
     doReturn(files).when(mockDir).listFiles();
     doReturn(mockDir).when(sd).getCurrentDir();
     
+
     return sd;
   }
   
@@ -170,11 +175,16 @@ public static FSEditLog createStandaloneEditLog(File logDir)
     assertTrue(logDir.mkdirs() || logDir.exists());
     Files.deleteDirectoryContents(logDir);
     NNStorage storage = Mockito.mock(NNStorage.class);
-    List<StorageDirectory> sds = Lists.newArrayList(
+    StorageDirectory sd 
-        FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS));
+      = FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS);
+    List<StorageDirectory> sds = Lists.newArrayList(sd);
     Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS);
+    Mockito.doReturn(sd).when(storage)
+      .getStorageDirectory(Matchers.<URI>anyObject());
 
-    return new FSEditLog(storage);
+    return new FSEditLog(new Configuration(), 
+                         storage,
+                         ImmutableList.of(logDir.toURI()));
   }
   
   /**

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java

@@ -46,6 +46,7 @@
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.common.Util;
 import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -861,8 +862,11 @@ public void testEditLogManifestMocks() throws IOException {
    * The syntax <code>[1,]</code> specifies an in-progress log starting at
    * txid 1.
    */
-  private NNStorage mockStorageWithEdits(String... editsDirSpecs) {
+  private NNStorage mockStorageWithEdits(String... editsDirSpecs) throws IOException {
     List<StorageDirectory> sds = Lists.newArrayList();
+    List<URI> uris = Lists.newArrayList();
+
+    NNStorage storage = Mockito.mock(NNStorage.class);
     for (String dirSpec : editsDirSpecs) {
       List<String> files = Lists.newArrayList();
       String[] logSpecs = dirSpec.split("\\|");
@@ -878,13 +882,17 @@ private NNStorage mockStorageWithEdits(String... editsDirSpecs) {
               Long.valueOf(m.group(2))));
         }
       }
-      sds.add(FSImageTestUtil.mockStorageDirectory(
+      StorageDirectory sd = FSImageTestUtil.mockStorageDirectory(
           NameNodeDirType.EDITS, false,
-          files.toArray(new String[0])));
+          files.toArray(new String[0]));
-    }
+      sds.add(sd);
-    
+      URI u = URI.create("file:///storage"+ Math.random());
-    NNStorage storage = Mockito.mock(NNStorage.class);
+      Mockito.doReturn(sd).when(storage).getStorageDirectory(u);
+      uris.add(u);
+    }    
+
     Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS);
+    Mockito.doReturn(uris).when(storage).getEditsDirectories();
     return storage;
   }