From e918b91e23985fa1bb353c54a2e733f8ba6dbe49 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 9 Feb 2012 06:12:01 +0000 Subject: [PATCH] HDFS-2579. Starting delegation token manager during safemode fails. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1242225 13f79535-47bb-0310-9956-ffa450edef68 --- .../AbstractDelegationTokenSecretManager.java | 24 +++- .../hadoop-hdfs/CHANGES.HDFS-1623.txt | 2 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../DelegationTokenSecretManager.java | 14 +- .../namenode/EditLogFileOutputStream.java | 5 + .../hdfs/server/namenode/FSNamesystem.java | 64 ++++++--- .../hadoop/fs/TestResolveHdfsSymlink.java | 5 +- .../fs/viewfs/TestViewFileSystemHdfs.java | 7 +- .../hadoop/fs/viewfs/TestViewFsHdfs.java | 6 +- .../apache/hadoop/hdfs/MiniDFSCluster.java | 4 + .../hdfs/security/TestDelegationToken.java | 56 +++++++- .../TestDelegationTokenForProxyUser.java | 4 +- .../namenode/OfflineEditsViewerHelper.java | 3 +- .../TestCheckPointForSecurityTokens.java | 7 +- .../namenode/TestSecurityTokenEditLog.java | 5 +- .../namenode/ha/TestHAStateTransitions.java | 126 +++++++++++++++++- 16 files changed, 295 insertions(+), 39 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index 3c2e666a39..11df9811b2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -40,6 +40,8 @@ import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.util.Daemon; +import com.google.common.base.Preconditions; + @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving public abstract @@ -84,6 +86,12 @@ class AbstractDelegationTokenSecretManager[] blocksToInject) public void setLeasePeriod(long soft, long hard) { NameNodeAdapter.setLeasePeriod(getNamesystem(), soft, hard); } + + public void setWaitSafeMode(boolean wait) { + this.waitSafeMode = wait; + } /** * Returns the current set of datanodes diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java index 4d18e98d1d..c2aaf0615c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java @@ -20,6 +20,8 @@ +import static org.junit.Assert.*; + import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; @@ -32,12 +34,16 @@ import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; @@ -64,6 +70,7 @@ public void setUp() throws Exception { config.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); config.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); config.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); + config.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); config.set("hadoop.security.auth_to_local", "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0"); @@ -71,7 +78,6 @@ public void setUp() throws Exception { cluster.waitActive(); dtSecretManager = NameNodeAdapter.getDtSecretManager( cluster.getNamesystem()); - dtSecretManager.startThreads(); } @After @@ -269,5 +275,51 @@ public Object run() throws IOException { } }); } - + + /** + * Test that the delegation token secret manager only runs when the + * NN is out of safe mode. This is because the secret manager + * has to log to the edit log, which should not be written in + * safe mode. Regression test for HDFS-2579. + */ + @Test + public void testDTManagerInSafeMode() throws Exception { + cluster.startDataNodes(config, 1, true, StartupOption.REGULAR, null); + FileSystem fs = cluster.getFileSystem(); + for (int i = 0; i < 5; i++) { + DFSTestUtil.createFile(fs, new Path("/test-" + i), 100, (short)1, 1L); + } + cluster.getConfiguration(0).setInt( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY, 500); + cluster.getConfiguration(0).setInt( + DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 30000); + cluster.setWaitSafeMode(false); + cluster.restartNameNode(); + NameNode nn = cluster.getNameNode(); + assertTrue(nn.isInSafeMode()); + DelegationTokenSecretManager sm = + NameNodeAdapter.getDtSecretManager(nn.getNamesystem()); + assertFalse("Secret manager should not run in safe mode", sm.isRunning()); + + NameNodeAdapter.leaveSafeMode(nn, false); + assertTrue("Secret manager should start when safe mode is exited", + sm.isRunning()); + + LOG.info("========= entering safemode again"); + + NameNodeAdapter.enterSafeMode(nn, false); + assertFalse("Secret manager should stop again when safe mode " + + "is manually entered", sm.isRunning()); + + // Set the cluster to leave safemode quickly on its own. + cluster.getConfiguration(0).setInt( + DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); + cluster.setWaitSafeMode(true); + cluster.restartNameNode(); + nn = cluster.getNameNode(); + sm = NameNodeAdapter.getDtSecretManager(nn.getNamesystem()); + + assertFalse(nn.isInSafeMode()); + assertTrue(sm.isRunning()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java index cdad31cc9b..6837f65afc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java @@ -48,7 +48,6 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; @@ -114,11 +113,12 @@ public void setUp() throws Exception { DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); config.setStrings(ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER), "group1"); + config.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); configureSuperUserIPAddresses(config, REAL_USER); FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0"); cluster = new MiniDFSCluster.Builder(config).build(); cluster.waitActive(); - NameNodeAdapter.getDtSecretManager(cluster.getNamesystem()).startThreads(); ProxyUsers.refreshSuperUserGroupsConfiguration(config); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java index e22fa29927..392cc9dd91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java @@ -108,10 +108,11 @@ public void startCluster(String dfsDir) throws IOException { // for security to work (fake JobTracker user) config.set("hadoop.security.auth_to_local", "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); + config.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); cluster = new MiniDFSCluster.Builder(config).manageNameDfsDirs(false).build(); cluster.waitClusterUp(); - cluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java index 1ba527702b..20d4c720de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java @@ -22,6 +22,7 @@ import java.io.*; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -64,11 +65,12 @@ public void testSaveNamespace() throws IOException { DistributedFileSystem fs = null; try { Configuration conf = new HdfsConfiguration(); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build(); cluster.waitActive(); fs = (DistributedFileSystem)(cluster.getFileSystem()); FSNamesystem namesystem = cluster.getNamesystem(); - namesystem.getDelegationTokenSecretManager().startThreads(); String renewer = UserGroupInformation.getLoginUser().getUserName(); Token token1 = namesystem .getDelegationToken(new Text(renewer)); @@ -122,7 +124,6 @@ public void testSaveNamespace() throws IOException { } namesystem = cluster.getNamesystem(); - namesystem.getDelegationTokenSecretManager().startThreads(); Token token3 = namesystem .getDelegationToken(new Text(renewer)); Token token4 = namesystem @@ -136,7 +137,6 @@ public void testSaveNamespace() throws IOException { cluster.waitActive(); namesystem = cluster.getNamesystem(); - namesystem.getDelegationTokenSecretManager().startThreads(); Token token5 = namesystem .getDelegationToken(new Text(renewer)); @@ -159,7 +159,6 @@ public void testSaveNamespace() throws IOException { cluster.waitActive(); namesystem = cluster.getNamesystem(); - namesystem.getDelegationTokenSecretManager().startThreads(); try { renewToken(token1); cancelToken(token1); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java index c0012be5ba..596df8d76b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -91,6 +92,9 @@ public void testEditLog() throws IOException { FileSystem fileSys = null; try { + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build(); cluster.waitActive(); fileSys = cluster.getFileSystem(); @@ -106,7 +110,6 @@ public void testEditLog() throws IOException { // set small size of flush buffer editLog.setOutputBufferCapacity(2048); - namesystem.getDelegationTokenSecretManager().startThreads(); // Create threads and make them run transactions concurrently. Thread threadId[] = new Thread[NUM_THREADS]; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java index 97a88d1e73..2595621641 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java @@ -317,6 +317,9 @@ public void testLeasesRenewedOnTransition() throws Exception { public void testDelegationTokensAfterFailover() throws IOException, URISyntaxException { Configuration conf = new Configuration(); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) .numDataNodes(0) @@ -326,7 +329,6 @@ public void testDelegationTokensAfterFailover() throws IOException, cluster.transitionToActive(0); NameNode nn1 = cluster.getNameNode(0); NameNode nn2 = cluster.getNameNode(1); - NameNodeAdapter.getDtSecretManager(nn1.getNamesystem()).startThreads(); String renewer = UserGroupInformation.getLoginUser().getUserName(); Token token = nn1.getRpcServer() @@ -335,8 +337,6 @@ public void testDelegationTokensAfterFailover() throws IOException, LOG.info("Failing over to NN 1"); cluster.transitionToStandby(0); cluster.transitionToActive(1); - // Need to explicitly start threads because security is not enabled. - NameNodeAdapter.getDtSecretManager(nn2.getNamesystem()).startThreads(); nn2.getRpcServer().renewDelegationToken(token); nn2.getRpcServer().cancelDelegationToken(token); @@ -421,4 +421,124 @@ private static void createEmptyInProgressEditLog(MiniDFSCluster cluster, EditLogFileOutputStream.writeHeader(out); } } + + + /** + * The secret manager needs to start/stop - the invariant should be that + * the secret manager runs if and only if the NN is active and not in + * safe mode. As a state diagram, we need to test all of the following + * transitions to make sure the secret manager is started when we transition + * into state 4, but none of the others. + * 
+   *         SafeMode     Not SafeMode 
+   * Standby   1 <------> 2
+   *           ^          ^
+   *           |          |
+   *           v          v
+   * Active    3 <------> 4
+   *
+ */ + @Test(timeout=60000) + public void testSecretManagerState() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + conf.setInt( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY, 50); + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHATopology()) + .numDataNodes(1) + .waitSafeMode(false) + .build(); + try { + cluster.transitionToActive(0); + DFSTestUtil.createFile(cluster.getFileSystem(0), + TEST_FILE_PATH, 6000, (short)1, 1L); + + cluster.getConfiguration(0).setInt( + DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 60000); + + cluster.restartNameNode(0); + NameNode nn = cluster.getNameNode(0); + + banner("Started in state 1."); + assertTrue(nn.isStandbyState()); + assertTrue(nn.isInSafeMode()); + assertFalse(isDTRunning(nn)); + + banner("Transition 1->2. Should not start secret manager"); + NameNodeAdapter.leaveSafeMode(nn, false); + assertTrue(nn.isStandbyState()); + assertFalse(nn.isInSafeMode()); + assertFalse(isDTRunning(nn)); + + banner("Transition 2->1. Should not start secret manager."); + NameNodeAdapter.enterSafeMode(nn, false); + assertTrue(nn.isStandbyState()); + assertTrue(nn.isInSafeMode()); + assertFalse(isDTRunning(nn)); + + banner("Transition 1->3. Should not start secret manager."); + nn.getRpcServer().transitionToActive(); + assertFalse(nn.isStandbyState()); + assertTrue(nn.isInSafeMode()); + assertFalse(isDTRunning(nn)); + + banner("Transition 3->1. Should not start secret manager."); + nn.getRpcServer().transitionToStandby(); + assertTrue(nn.isStandbyState()); + assertTrue(nn.isInSafeMode()); + assertFalse(isDTRunning(nn)); + + banner("Transition 1->3->4. Should start secret manager."); + nn.getRpcServer().transitionToActive(); + NameNodeAdapter.leaveSafeMode(nn, false); + assertFalse(nn.isStandbyState()); + assertFalse(nn.isInSafeMode()); + assertTrue(isDTRunning(nn)); + + banner("Transition 4->3. Should stop secret manager"); + NameNodeAdapter.enterSafeMode(nn, false); + assertFalse(nn.isStandbyState()); + assertTrue(nn.isInSafeMode()); + assertFalse(isDTRunning(nn)); + + banner("Transition 3->4. Should start secret manager"); + NameNodeAdapter.leaveSafeMode(nn, false); + assertFalse(nn.isStandbyState()); + assertFalse(nn.isInSafeMode()); + assertTrue(isDTRunning(nn)); + + for (int i = 0; i < 20; i++) { + // Loop the last check to suss out races. + banner("Transition 4->2. Should stop secret manager."); + nn.getRpcServer().transitionToStandby(); + assertTrue(nn.isStandbyState()); + assertFalse(nn.isInSafeMode()); + assertFalse(isDTRunning(nn)); + + banner("Transition 2->4. Should start secret manager"); + nn.getRpcServer().transitionToActive(); + assertFalse(nn.isStandbyState()); + assertFalse(nn.isInSafeMode()); + assertTrue(isDTRunning(nn)); + } + } finally { + cluster.shutdown(); + } + } + + private boolean isDTRunning(NameNode nn) { + return NameNodeAdapter.getDtSecretManager(nn.getNamesystem()).isRunning(); + } + + /** + * Print a big banner in the test log to make debug easier. + */ + static void banner(String string) { + LOG.info("\n\n\n\n================================================\n" + + string + "\n" + + "==================================================\n\n"); + } }