From e17d5a96833410478e95b8f32ccb64ae436f89ec Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 29 Apr 2013 02:07:58 +0000 Subject: [PATCH] HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows. Contributed by Arpit Agarwal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1476877 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/tools/TestDFSHAAdmin.java | 46 ++++++++++++++----- .../hdfs/tools/TestDFSHAAdminMiniCluster.java | 30 ++++++++---- 3 files changed, 58 insertions(+), 21 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 15d7b00a64..fcb437eb10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -326,6 +326,9 @@ Trunk (Unreleased) HDFS-4705. Address HDFS test failures on Windows because of invalid dfs.namenode.name.dir. (Ivan Mitic via suresh) + HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows. + (Arpit Agarwal via suresh) + BREAKDOWN OF HDFS-347 SUBTASKS AND RELATED JIRAS HDFS-4353. Encapsulate connections to peers in Peer and PeerServer classes. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java index 666e52b484..c9e4665203 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.test.MockitoUtil; +import org.apache.hadoop.util.Shell; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -73,6 +74,17 @@ public class TestDFSHAAdmin { private static String HOST_A = "1.2.3.1"; private static String HOST_B = "1.2.3.2"; + // Fencer shell commands that always return true and false respectively + // on Unix. + private static String FENCER_TRUE_COMMAND_UNIX = "shell(true)"; + private static String FENCER_FALSE_COMMAND_UNIX = "shell(false)"; + + // Fencer shell commands that always return true and false respectively + // on Windows. Lacking POSIX 'true' and 'false' commands we use the DOS + // commands 'rem' and 'help.exe'. + private static String FENCER_TRUE_COMMAND_WINDOWS = "shell(rem)"; + private static String FENCER_FALSE_COMMAND_WINDOWS = "shell(help.exe /? >NUL)"; + private HdfsConfiguration getHAConf() { HdfsConfiguration conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID); @@ -89,6 +101,16 @@ private HdfsConfiguration getHAConf() { return conf; } + public static String getFencerTrueCommand() { + return Shell.WINDOWS ? + FENCER_TRUE_COMMAND_WINDOWS : FENCER_TRUE_COMMAND_UNIX; + } + + public static String getFencerFalseCommand() { + return Shell.WINDOWS ? + FENCER_FALSE_COMMAND_WINDOWS : FENCER_FALSE_COMMAND_UNIX; + } + @Before public void setup() throws IOException { mockProtocol = MockitoUtil.mockProtocol(HAServiceProtocol.class); @@ -173,7 +195,7 @@ public void testMutativeOperationsWithAutoHaEnabled() throws Exception { // Turn on auto-HA in the config HdfsConfiguration conf = getHAConf(); conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand()); tool.setConf(conf); // Should fail without the forcemanual flag @@ -250,7 +272,7 @@ public void testFailoverWithNoFencerConfigured() throws Exception { public void testFailoverWithFencerConfigured() throws Exception { Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus(); HdfsConfiguration conf = getHAConf(); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand()); tool.setConf(conf); assertEquals(0, runTool("-failover", "nn1", "nn2")); } @@ -259,7 +281,7 @@ public void testFailoverWithFencerConfigured() throws Exception { public void testFailoverWithFencerAndNameservice() throws Exception { Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus(); HdfsConfiguration conf = getHAConf(); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand()); tool.setConf(conf); assertEquals(0, runTool("-ns", "ns1", "-failover", "nn1", "nn2")); } @@ -268,7 +290,7 @@ public void testFailoverWithFencerAndNameservice() throws Exception { public void testFailoverWithFencerConfiguredAndForce() throws Exception { Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus(); HdfsConfiguration conf = getHAConf(); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand()); tool.setConf(conf); assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence")); } @@ -277,7 +299,7 @@ public void testFailoverWithFencerConfiguredAndForce() throws Exception { public void testFailoverWithForceActive() throws Exception { Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus(); HdfsConfiguration conf = getHAConf(); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand()); tool.setConf(conf); assertEquals(0, runTool("-failover", "nn1", "nn2", "--forceactive")); } @@ -286,7 +308,7 @@ public void testFailoverWithForceActive() throws Exception { public void testFailoverWithInvalidFenceArg() throws Exception { Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus(); HdfsConfiguration conf = getHAConf(); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand()); tool.setConf(conf); assertEquals(-1, runTool("-failover", "nn1", "nn2", "notforcefence")); } @@ -312,7 +334,7 @@ public void testFailoverWithAutoHa() throws Exception { // Turn on auto-HA in the config HdfsConfiguration conf = getHAConf(); conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand()); tool.setConf(conf); assertEquals(0, runTool("-failover", "nn1", "nn2")); @@ -323,7 +345,7 @@ public void testFailoverWithAutoHa() throws Exception { public void testForceFenceOptionListedBeforeArgs() throws Exception { Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus(); HdfsConfiguration conf = getHAConf(); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand()); tool.setConf(conf); assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2")); } @@ -359,23 +381,23 @@ public void testFencingConfigPerNameNode() throws Exception { HdfsConfiguration conf = getHAConf(); // Set the default fencer to succeed - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand()); tool.setConf(conf); assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence")); // Set the NN-specific fencer to fail. Should fail to fence. - conf.set(nnSpecificKey, "shell(false)"); + conf.set(nnSpecificKey, getFencerFalseCommand()); tool.setConf(conf); assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence")); conf.unset(nnSpecificKey); // Set an NS-specific fencer to fail. Should fail. - conf.set(nsSpecificKey, "shell(false)"); + conf.set(nsSpecificKey, getFencerFalseCommand()); tool.setConf(conf); assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence")); // Set the NS-specific fencer to succeed. Should succeed - conf.set(nsSpecificKey, "shell(true)"); + conf.set(nsSpecificKey, getFencerTrueCommand()); tool.setConf(conf); assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence")); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java index 38380b55f9..dc6b348656 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.util.Shell; import org.apache.log4j.Level; import org.junit.After; import org.junit.Before; @@ -114,7 +115,8 @@ public void testStateTransition() throws Exception { @Test public void testTryFailoverToSafeMode() throws Exception { - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, + TestDFSHAAdmin.getFencerTrueCommand()); tool.setConf(conf); NameNodeAdapter.enterSafeMode(cluster.getNameNode(0), false); @@ -136,10 +138,17 @@ public void testFencer() throws Exception { // tmp file, so we can verify that the args were substituted right File tmpFile = File.createTempFile("testFencer", ".txt"); tmpFile.deleteOnExit(); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, - "shell(echo -n $target_nameserviceid.$target_namenodeid " + - "$target_port $dfs_ha_namenode_id > " + - tmpFile.getAbsolutePath() + ")"); + if (Shell.WINDOWS) { + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, + "shell(echo %target_nameserviceid%.%target_namenodeid% " + + "%target_port% %dfs_ha_namenode_id% > " + + tmpFile.getAbsolutePath() + ")"); + } else { + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, + "shell(echo -n $target_nameserviceid.$target_namenodeid " + + "$target_port $dfs_ha_namenode_id > " + + tmpFile.getAbsolutePath() + ")"); + } // Test failover with fencer tool.setConf(conf); @@ -156,9 +165,11 @@ public void testFencer() throws Exception { assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence")); // The fence script should run with the configuration from the target - // node, rather than the configuration from the fencing node - assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1", - Files.toString(tmpFile, Charsets.UTF_8)); + // node, rather than the configuration from the fencing node. Strip + // out any trailing spaces and CR/LFs which may be present on Windows. + String fenceCommandOutput =Files.toString(tmpFile, Charsets.UTF_8). + replaceAll(" *[\r\n]+", ""); + assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1", fenceCommandOutput); tmpFile.delete(); // Test failover with forceactive option @@ -181,7 +192,8 @@ public void testFencer() throws Exception { assertFalse(tmpFile.exists()); // Test failover with force fence listed before the other arguments - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)"); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, + TestDFSHAAdmin.getFencerTrueCommand()); tool.setConf(conf); assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2")); }