From 969318cfc2397110a686a0a728b56b6a0cab9f4d Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 3 Feb 2012 18:04:21 +0000 Subject: [PATCH] HDFS-2863. Failures observed if dfs.edits.dir and shared.edits.dir have same directories. Contributed by Bikas Saha. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1240267 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES.HDFS-1623.txt | 2 + .../hdfs/server/namenode/FSNamesystem.java | 14 ++++++ .../server/namenode/TestFSNamesystem.java | 47 +++++++++++++++++++ .../namenode/ha/TestHAConfiguration.java | 22 ++++++++- 4 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt index 9e24b2071f..92e634bb9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt @@ -158,3 +158,5 @@ HDFS-2860. TestDFSRollback#testRollback is failing. (atm) HDFS-2769. HA: When HA is enabled with a shared edits dir, that dir should be marked required. (atm via eli) + +HDFS-2863. Failures observed if dfs.edits.dir and shared.edits.dir have same directories. (Bikas Saha via atm) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 7754a9085b..c0f8a82e61 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -190,6 +190,7 @@ import org.mortbay.util.ajax.JSON; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.annotations.VisibleForTesting; @@ -680,6 +681,19 @@ private static Collection getStorageDirs(Configuration conf, public static Collection getNamespaceEditsDirs(Configuration conf) { Collection editsDirs = getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_KEY); editsDirs.addAll(getSharedEditsDirs(conf)); + Set uniqueEditsDirs = new HashSet(); + uniqueEditsDirs.addAll(editsDirs); + if (uniqueEditsDirs.size() != editsDirs.size()) { + // clearing and re-initializing editsDirs to preserve Collection semantics + // assigning finalEditsDirs to editsDirs would leak Set semantics in the + // return value and cause unexpected results downstream. eg future addAll + // calls. Perf is not an issue since these are small lists. + editsDirs.clear(); + editsDirs.addAll(uniqueEditsDirs); + LOG.warn("Overlapping entries in " + DFS_NAMENODE_EDITS_DIR_KEY + + " and/or " + DFS_NAMENODE_SHARED_EDITS_DIR_KEY + + ". Using the following entries: " + Joiner.on(',').join(editsDirs)); + } if (editsDirs.isEmpty()) { // If this is the case, no edit dirs have been explicitly configured. // Image dirs are to be used for edits too. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java new file mode 100644 index 0000000000..d79f583efd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.*; +import static org.junit.Assert.*; + +import java.net.URI; +import java.util.Collection; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; + +public class TestFSNamesystem { + + /** + * Tests that the namenode edits dirs are gotten with duplicates removed + */ + @Test + public void testUniqueEditDirs() { + Configuration config = new Configuration(); + + config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, " + + "file://edits/dir1,file://edits/dir1"); // overlapping internally + + // getNamespaceEditsDirs removes duplicates + Collection editsDirs = FSNamesystem.getNamespaceEditsDirs(config); + assertEquals(2, editsDirs.size()); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java index 89d7ddd084..12472c4f10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java @@ -17,8 +17,11 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; +import static org.apache.hadoop.hdfs.DFSConfigKeys.*; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; + +import java.net.URI; +import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -78,4 +81,21 @@ public void testGetOtherNNHttpAddress() { assertEquals(HOST_B + ":" + DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT, checkpointer.getActiveNNAddress()); } + + /** + * Tests that the namenode edits dirs and shared edits dirs are gotten with + * duplicates removed + */ + @Test + public void testHAUniqueEditDirs() { + Configuration config = new Configuration(); + + config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, " + + "file://edits/shared/dir"); // overlapping + config.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, "file://edits/shared/dir"); + + // getNamespaceEditsDirs removes duplicates across edits and shared.edits + Collection editsDirs = FSNamesystem.getNamespaceEditsDirs(config); + assertEquals(2, editsDirs.size()); + } }