HDFS-2863. Failures observed if dfs.edits.dir and shared.edits.dir have same directories. Contributed by Bikas Saha.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1240267 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt

@@ -158,3 +158,5 @@ HDFS-2860. TestDFSRollback#testRollback is failing. (atm)
 
 HDFS-2769. HA: When HA is enabled with a shared edits dir, that dir should be
 marked required. (atm via eli)
+
+HDFS-2863. Failures observed if dfs.edits.dir and shared.edits.dir have same directories. (Bikas Saha via atm)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -190,6 +190,7 @@
 import org.mortbay.util.ajax.JSON;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -680,6 +681,19 @@ private static Collection<URI> getStorageDirs(Configuration conf,
   public static Collection<URI> getNamespaceEditsDirs(Configuration conf) {
     Collection<URI> editsDirs = getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_KEY);
     editsDirs.addAll(getSharedEditsDirs(conf));
+    Set<URI> uniqueEditsDirs = new HashSet<URI>();
+    uniqueEditsDirs.addAll(editsDirs);
+    if (uniqueEditsDirs.size() != editsDirs.size()) {
+      // clearing and re-initializing editsDirs to preserve Collection semantics
+      // assigning finalEditsDirs to editsDirs would leak Set semantics in the 
+      // return value and cause unexpected results downstream. eg future addAll
+      // calls. Perf is not an issue since these are small lists.
+      editsDirs.clear();
+      editsDirs.addAll(uniqueEditsDirs);
+      LOG.warn("Overlapping entries in " + DFS_NAMENODE_EDITS_DIR_KEY 
+          + " and/or " + DFS_NAMENODE_SHARED_EDITS_DIR_KEY
+          + ". Using the following entries: " + Joiner.on(',').join(editsDirs));
+    }
     if (editsDirs.isEmpty()) {
       // If this is the case, no edit dirs have been explicitly configured.
       // Image dirs are to be used for edits too.

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.junit.Assert.*;
+
+import java.net.URI;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+public class TestFSNamesystem {
+
+  /**
+   * Tests that the namenode edits dirs are gotten with duplicates removed
+   */
+  @Test
+  public void testUniqueEditDirs() {
+    Configuration config = new Configuration();
+
+    config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "
+        + "file://edits/dir1,file://edits/dir1"); // overlapping internally
+
+    // getNamespaceEditsDirs removes duplicates
+    Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(config);
+    assertEquals(2, editsDirs.size());
+  }
+
+}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java

@@ -17,8 +17,11 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.ha;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
 import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
+
+import java.net.URI;
+import java.util.Collection;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -78,4 +81,21 @@ public void testGetOtherNNHttpAddress() {
     assertEquals(HOST_B + ":" + DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT,
         checkpointer.getActiveNNAddress());
   }
+  
+  /**
+   * Tests that the namenode edits dirs and shared edits dirs are gotten with
+   * duplicates removed
+   */
+  @Test
+  public void testHAUniqueEditDirs() {
+    Configuration config = new Configuration();
+
+    config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "
+        + "file://edits/shared/dir"); // overlapping
+    config.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, "file://edits/shared/dir");
+
+    // getNamespaceEditsDirs removes duplicates across edits and shared.edits
+    Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(config);
+    assertEquals(2, editsDirs.size());
+  }
 }