diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt index c8a760336a..044181097f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt @@ -133,3 +133,5 @@ HDFS-2807. Service level authorizartion for HAServiceProtocol. (jitendra) HDFS-2809. Add test to verify that delegation tokens are honored after failover. (jitendra and atm) HDFS-2838. NPE in FSNamesystem when in safe mode. (Gregory Chanan via eli) + +HDFS-2805. Add a test for a federated cluster with HA NNs. (Brandon Li via jitendra) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 977ee956cb..bf3af609d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -586,11 +586,19 @@ private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, conf.set(FS_DEFAULT_NAME_KEY, "127.0.0.1:" + onlyNN.getIpcPort()); } + // If we have more than one nameservice, need to enumerate them in the + // config. + if (federation) { + List allNsIds = Lists.newArrayList(); + for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) { + allNsIds.add(nameservice.getId()); + } + conf.set(DFS_FEDERATION_NAMESERVICES, Joiner.on(",").join(allNsIds)); + } + int nnCounter = 0; - List nsIds = Lists.newArrayList(); for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) { String nsId = nameservice.getId(); - nsIds.add(nameservice.getId()); Preconditions.checkArgument( !federation || nsId != null, @@ -643,6 +651,7 @@ private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, } prevNNDirs = FSNamesystem.getNamespaceDirs(conf); } + // Start all Namenodes for (NNConf nn : nameservice.getNNs()) { initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs, nnCounter); @@ -651,12 +660,7 @@ private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, } } - if (federation) { - // If we have more than one nameservice, need to enumerate them in the - // config. - conf.set(DFS_FEDERATION_NAMESERVICES, Joiner.on(",").join(nsIds)); - } - + } public URI getSharedEditsDir(int minNN, int maxNN) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java index c8e22e3b45..311e687526 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java @@ -77,6 +77,22 @@ public static MiniDFSNNTopology simpleFederatedTopology( return topology; } + /** + * Set up federated cluster with the given number of nameservices, each + * of which has two NameNodes. + */ + public static MiniDFSNNTopology simpleHAFederatedTopology( + int numNameservices) { + MiniDFSNNTopology topology = new MiniDFSNNTopology(); + for (int i = 0; i < numNameservices; i++) { + topology.addNameservice(new MiniDFSNNTopology.NSConf("ns" + i) + .addNN(new MiniDFSNNTopology.NNConf("nn0")) + .addNN(new MiniDFSNNTopology.NNConf("nn1"))); + } + topology.setFederation(true); + return topology; + } + public MiniDFSNNTopology setFederation(boolean federation) { this.federation = federation; return this; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java index 5536ba37b5..5439d15b81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java @@ -131,18 +131,36 @@ public CouldNotCatchUpException(String message) { /** Gets the filesystem instance by setting the failover configurations */ public static FileSystem configureFailoverFs(MiniDFSCluster cluster, Configuration conf) throws IOException, URISyntaxException { + return configureFailoverFs(cluster, conf, 0); + } + + /** + * Gets the filesystem instance by setting the failover configurations + * @param cluster the single process DFS cluster + * @param conf cluster configuration + * @param nsIndex namespace index starting with zero + * @throws IOException if an error occurs rolling the edit log + */ + public static FileSystem configureFailoverFs(MiniDFSCluster cluster, Configuration conf, + int nsIndex) throws IOException, URISyntaxException { conf = new Configuration(conf); String logicalName = getLogicalHostname(cluster); - setFailoverConfigurations(cluster, conf, logicalName); + setFailoverConfigurations(cluster, conf, logicalName, nsIndex); FileSystem fs = FileSystem.get(new URI("hdfs://" + logicalName), conf); return fs; } - - /** Sets the required configurations for performing failover */ + + /** Sets the required configurations for performing failover of default namespace. */ public static void setFailoverConfigurations(MiniDFSCluster cluster, Configuration conf, String logicalName) { - InetSocketAddress nnAddr1 = cluster.getNameNode(0).getNameNodeAddress(); - InetSocketAddress nnAddr2 = cluster.getNameNode(1).getNameNodeAddress(); + setFailoverConfigurations(cluster, conf, logicalName, 0); + } + + /** Sets the required configurations for performing failover. */ + public static void setFailoverConfigurations(MiniDFSCluster cluster, + Configuration conf, String logicalName, int nsIndex) { + InetSocketAddress nnAddr1 = cluster.getNameNode(2 * nsIndex).getNameNodeAddress(); + InetSocketAddress nnAddr2 = cluster.getNameNode(2 * nsIndex + 1).getNameNodeAddress(); String nameNodeId1 = "nn1"; String nameNodeId2 = "nn2"; String address1 = "hdfs://" + nnAddr1.getHostName() + ":" + nnAddr1.getPort(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java index 5197c6e764..3b31445f68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java @@ -106,6 +106,46 @@ public void testTransitionActiveToStandby() throws Exception { } } + /** + * Test manual failover failback for one namespace + * @param cluster single process test cluster + * @param conf cluster configuration + * @param nsIndex namespace index starting from zero + * @throws Exception + */ + private void testManualFailoverFailback(MiniDFSCluster cluster, + Configuration conf, int nsIndex) throws Exception { + int nn0 = 2 * nsIndex, nn1 = 2 * nsIndex + 1; + + cluster.transitionToActive(nn0); + + LOG.info("Starting with NN 0 active in namespace " + nsIndex); + FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); + fs.mkdirs(TEST_DIR); + + LOG.info("Failing over to NN 1 in namespace " + nsIndex); + cluster.transitionToStandby(nn0); + cluster.transitionToActive(nn1); + assertTrue(fs.exists(TEST_DIR)); + DFSTestUtil.writeFile(fs, TEST_FILE_PATH, TEST_FILE_DATA); + + LOG.info("Failing over to NN 0 in namespace " + nsIndex); + cluster.transitionToStandby(nn1); + cluster.transitionToActive(nn0); + assertTrue(fs.exists(TEST_DIR)); + assertEquals(TEST_FILE_DATA, + DFSTestUtil.readFile(fs, TEST_FILE_PATH)); + + LOG.info("Removing test file"); + fs.delete(TEST_DIR, true); + assertFalse(fs.exists(TEST_DIR)); + + LOG.info("Failing over to NN 1 in namespace " + nsIndex); + cluster.transitionToStandby(nn0); + cluster.transitionToActive(nn1); + assertFalse(fs.exists(TEST_DIR)); + } + /** * Tests manual failover back and forth between two NameNodes. */ @@ -118,34 +158,8 @@ public void testManualFailoverAndFailback() throws Exception { .build(); try { cluster.waitActive(); - cluster.transitionToActive(0); - - LOG.info("Starting with NN 0 active"); - FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); - fs.mkdirs(TEST_DIR); - - LOG.info("Failing over to NN 1"); - cluster.transitionToStandby(0); - cluster.transitionToActive(1); - assertTrue(fs.exists(TEST_DIR)); - DFSTestUtil.writeFile(fs, TEST_FILE_PATH, TEST_FILE_DATA); - - LOG.info("Failing over to NN 0"); - cluster.transitionToStandby(1); - cluster.transitionToActive(0); - assertTrue(fs.exists(TEST_DIR)); - assertEquals(TEST_FILE_DATA, - DFSTestUtil.readFile(fs, TEST_FILE_PATH)); - - LOG.info("Removing test file"); - fs.delete(TEST_DIR, true); - assertFalse(fs.exists(TEST_DIR)); - - LOG.info("Failing over to NN 1"); - cluster.transitionToStandby(0); - cluster.transitionToActive(1); - assertFalse(fs.exists(TEST_DIR)); - + // test the only namespace + testManualFailoverFailback(cluster, conf, 0); } finally { cluster.shutdown(); } @@ -294,4 +308,28 @@ public void testDelegationTokensAfterFailover() throws IOException, cluster.shutdown(); } } + + /** + * Tests manual failover back and forth between two NameNodes + * for federation cluster with two namespaces. + */ + @Test + public void testManualFailoverFailbackFederationHA() throws Exception { + Configuration conf = new Configuration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHAFederatedTopology(2)) + .numDataNodes(1) + .build(); + try { + cluster.waitActive(); + + // test for namespace 0 + testManualFailoverFailback(cluster, conf, 0); + + // test for namespace 1 + testManualFailoverFailback(cluster, conf, 1); + } finally { + cluster.shutdown(); + } + } }