big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-2752. HA: exit if multiple shared dirs are configured. Contributed by Eli Collins

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1240916 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-02-06 08:24:11 +00:00
parent c17b4f8eef
commit 2e4cf977ae
6 changed files with 45 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
View File

@ -170,3 +170,5 @@ HDFS-2792. Make fsck work. (atm)
HDFS-2808. HA: haadmin should use namenode ids. (eli)
HDFS-2819. Document new HA-related configs in hdfs-default.xml. (eli)
HDFS-2752. HA: exit if multiple shared dirs are configured. (eli)

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -683,17 +683,28 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
* are ordered before non-shared directories, and any duplicates
* are removed. The order they are specified in the configuration
* is retained.
* @return Collection of shared edits directories.
* @throws IOException if multiple shared edits directories are configured
*/
public static List<URI> getNamespaceEditsDirs(Configuration conf) {
public static List<URI> getNamespaceEditsDirs(Configuration conf)
throws IOException {
// Use a LinkedHashSet so that order is maintained while we de-dup
// the entries.
LinkedHashSet<URI> editsDirs = new LinkedHashSet<URI>();
List<URI> sharedDirs = getSharedEditsDirs(conf);
// Fail until multiple shared edits directories are supported (HDFS-2782)
if (sharedDirs.size() > 1) {
throw new IOException(
"Multiple shared edits directories are not yet supported");
}
// First add the shared edits dirs. It's critical that the shared dirs
// are added first, since JournalSet syncs them in the order they are listed,
// and we need to make sure all edits are in place in the shared storage
// before they are replicated locally. See HDFS-2874.
for (URI dir : getSharedEditsDirs(conf)) {
for (URI dir : sharedDirs) {
if (!editsDirs.add(dir)) {
LOG.warn("Edits URI " + dir + " listed multiple times in " +
DFS_NAMENODE_SHARED_EDITS_DIR_KEY + ". Ignoring duplicates.");

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -1594,7 +1594,7 @@ public class MiniDFSCluster {
/**
* Get the directories where the namenode stores its edits.
*/
public Collection<URI> getNameEditsDirs(int nnIndex) {
public Collection<URI> getNameEditsDirs(int nnIndex) throws IOException {
return FSNamesystem.getNamespaceEditsDirs(nameNodes[nnIndex].conf);
}

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.junit.Assert.*;
import java.io.IOException;
import java.net.URI;
import java.util.Collection;
@ -33,7 +34,7 @@ public class TestFSNamesystem {
* Tests that the namenode edits dirs are gotten with duplicates removed
*/
@Test
public void testUniqueEditDirs() {
public void testUniqueEditDirs() throws IOException {
Configuration config = new Configuration();
config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "

28
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
View File

@ -68,6 +68,29 @@ public class TestFailureOfSharedDir {
requiredEditsDirs.contains(bar));
}
/**
* Multiple shared edits directories is an invalid configuration.
*/
@Test
public void testMultipleSharedDirsFails() throws Exception {
Configuration conf = new Configuration();
URI sharedA = new URI("file:///shared-A");
URI sharedB = new URI("file:///shared-B");
URI localA = new URI("file:///local-A");
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
Joiner.on(",").join(sharedA,sharedB));
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
localA.toString());
try {
FSNamesystem.getNamespaceEditsDirs(conf);
fail("Allowed multiple shared edits directories");
} catch (IOException ioe) {
assertEquals("Multiple shared edits directories are not yet supported",
ioe.getMessage());
}
}
/**
* Make sure that the shared edits dirs are listed before non-shared dirs
@ -78,13 +101,12 @@ public class TestFailureOfSharedDir {
public void testSharedDirsComeFirstInEditsList() throws Exception {
Configuration conf = new Configuration();
URI sharedA = new URI("file:///shared-A");
URI sharedB = new URI("file:///shared-B");
URI localA = new URI("file:///local-A");
URI localB = new URI("file:///local-B");
URI localC = new URI("file:///local-C");
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
Joiner.on(",").join(sharedA,sharedB));
sharedA.toString());
// List them in reverse order, to make sure they show up in
// the order listed, regardless of lexical sort order.
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
@ -93,7 +115,7 @@ public class TestFailureOfSharedDir {
assertEquals(
"Shared dirs should come first, then local dirs, in the order " +
"they were listed in the configuration.",
Joiner.on(",").join(sharedA, sharedB, localC, localB, localA),
Joiner.on(",").join(sharedA, localC, localB, localA),
Joiner.on(",").join(dirs));
}

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java
View File

@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode.ha;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.junit.Assert.*;
import java.io.IOException;
import java.net.URI;
import java.util.Collection;
@ -87,7 +88,7 @@ public class TestHAConfiguration {
* duplicates removed
*/
@Test
public void testHAUniqueEditDirs() {
public void testHAUniqueEditDirs() throws IOException {
Configuration config = new Configuration();
config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "