HADOOP-10540. Datanode upgrade in Windows fails with hardlink error. (Contributed by Chris Nauroth and Arpit Agarwal)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1589923 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -416,6 +416,9 @@ Release 2.5.0 - UNRELEASED
     HADOOP-10539. Provide backward compatibility for ProxyUsers.authorize()
     call. (Benoy Antony via cnauroth)
 
+    HADOOP-10540. Datanode upgrade in Windows fails with hardlink error.
+    (Chris Nauroth and Arpit Agarwal)
+
 Release 2.4.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java

@@ -22,10 +22,13 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStreamReader;
+import java.io.StringReader;
 import java.util.Arrays;
 
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.Shell.ExitCodeException;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 
 /**
  * Class for creating hardlinks.
@@ -89,7 +92,6 @@ private static abstract class HardLinkCommandGetter {
      *            to the source directory
      * @param linkDir - target directory where the hardlinks will be put
      * @return - an array of Strings suitable for use as a single shell command
-     *            with {@link Runtime.exec()}
      * @throws IOException - if any of the file or path names misbehave
      */
     abstract String[] linkMult(String[] fileBaseNames, File linkDir) 
@@ -230,17 +232,17 @@ static class HardLinkCGWin extends HardLinkCommandGetter {
     //package-private ("default") access instead of "private" to assist 
     //unit testing (sort of) on non-Win servers
 
+    static String CMD_EXE = "cmd.exe";
     static String[] hardLinkCommand = {
                         Shell.WINUTILS,"hardlink","create", null, null};
     static String[] hardLinkMultPrefix = {
-                        "cmd","/q","/c","for", "%f", "in", "("};
+        CMD_EXE, "/q", "/c", "for", "%f", "in", "("};
     static String   hardLinkMultDir = "\\%f";
     static String[] hardLinkMultSuffix = {
-                        ")", "do", Shell.WINUTILS, "hardlink", "create", null,
+        ")", "do", Shell.WINUTILS, "hardlink", "create", null,
-                        "%f", "1>NUL"};
+        "%f"};
     static String[] getLinkCountCommand = {
-                        Shell.WINUTILS, "hardlink",
+        Shell.WINUTILS, "hardlink", "stat", null};
-                        "stat", null};
     //Windows guarantees only 8K - 1 bytes cmd length.
     //Subtract another 64b to allow for Java 'exec' overhead
     static final int maxAllowedCmdArgLength = 8*1024 - 65;
@@ -278,7 +280,7 @@ String[] linkMult(String[] fileBaseNames, File linkDir)
       System.arraycopy(hardLinkMultSuffix, 0, buf, mark, 
                        hardLinkMultSuffix.length);
       mark += hardLinkMultSuffix.length;
-      buf[mark - 3] = td;
+      buf[mark - 2] = td;
       return buf;
     }
     
@@ -310,8 +312,8 @@ int getLinkMultArgLength(File fileDir, String[] fileBaseNames, File linkDir)
                linkDir.getCanonicalPath().length();
       //add the fixed overhead of the hardLinkMult command 
       //(prefix, suffix, and Dir suffix)
-      sum += ("cmd.exe /q /c for %f in ( ) do "
+      sum += (CMD_EXE + " /q /c for %f in ( ) do "
-              + Shell.WINUTILS + " hardlink create \\%f %f 1>NUL ").length();
+              + Shell.WINUTILS + " hardlink create \\%f %f").length();
       return sum;
     }
     
@@ -379,21 +381,14 @@ public static void createHardLink(File file, File linkName)
     }
 	  // construct and execute shell command
     String[] hardLinkCommand = getHardLinkCommand.linkOne(file, linkName);
-    Process process = Runtime.getRuntime().exec(hardLinkCommand);
+    ShellCommandExecutor shexec = new ShellCommandExecutor(hardLinkCommand);
     try {
-      if (process.waitFor() != 0) {
+      shexec.execute();
-        String errMsg = new BufferedReader(new InputStreamReader(
+    } catch (ExitCodeException e) {
-            process.getInputStream())).readLine();
+      throw new IOException("Failed to execute command " +
-        if (errMsg == null)  errMsg = "";
+          Arrays.toString(hardLinkCommand) +
-        String inpMsg = new BufferedReader(new InputStreamReader(
+          "; command output: \"" + shexec.getOutput() + "\"" +
-            process.getErrorStream())).readLine();
+          "; WrappedException: \"" + e.getMessage() + "\"");
-        if (inpMsg == null)  inpMsg = "";
-        throw new IOException(errMsg + inpMsg);
-      }
-    } catch (InterruptedException e) {
-      throw new IOException(e);
-    } finally {
-      process.destroy();
     }
   }
 
@@ -466,22 +461,12 @@ protected static int createHardLinkMult(File parentDir,
     // construct and execute shell command
     String[] hardLinkCommand = getHardLinkCommand.linkMult(fileBaseNames, 
         linkDir);
-    Process process = Runtime.getRuntime().exec(hardLinkCommand, null, 
+    ShellCommandExecutor shexec = new ShellCommandExecutor(hardLinkCommand,
-        parentDir);
+      parentDir, null, 0L);
     try {
-      if (process.waitFor() != 0) {
+      shexec.execute();
-        String errMsg = new BufferedReader(new InputStreamReader(
+    } catch (ExitCodeException e) {
-            process.getInputStream())).readLine();
+      throw new IOException(shexec.getOutput() + e.getMessage());
-        if (errMsg == null)  errMsg = "";
-        String inpMsg = new BufferedReader(new InputStreamReader(
-            process.getErrorStream())).readLine();
-        if (inpMsg == null)  inpMsg = "";
-        throw new IOException(errMsg + inpMsg);
-      }
-    } catch (InterruptedException e) {
-      throw new IOException(e);
-    } finally {
-      process.destroy();
     }
     return callCount;
   }
@@ -504,17 +489,13 @@ public static int getLinkCount(File fileName) throws IOException {
     String errMsg = null;
     int exitValue = -1;
     BufferedReader in = null;
-    BufferedReader err = null;
 
-    Process process = Runtime.getRuntime().exec(cmd);
+    ShellCommandExecutor shexec = new ShellCommandExecutor(cmd);
     try {
-      exitValue = process.waitFor();
+      shexec.execute();
-      in = new BufferedReader(new InputStreamReader(
+      in = new BufferedReader(new StringReader(shexec.getOutput()));
-                                  process.getInputStream()));
       inpMsg = in.readLine();
-      err = new BufferedReader(new InputStreamReader(
+      exitValue = shexec.getExitCode();
-                                   process.getErrorStream()));
-      errMsg = err.readLine();
       if (inpMsg == null || exitValue != 0) {
         throw createIOException(fileName, inpMsg, errMsg, exitValue, null);
       }
@@ -524,14 +505,15 @@ public static int getLinkCount(File fileName) throws IOException {
       } else {
         return Integer.parseInt(inpMsg);
       }
+    } catch (ExitCodeException e) {
+      inpMsg = shexec.getOutput();
+      errMsg = e.getMessage();
+      exitValue = e.getExitCode();
+      throw createIOException(fileName, inpMsg, errMsg, exitValue, e);
     } catch (NumberFormatException e) {
       throw createIOException(fileName, inpMsg, errMsg, exitValue, e);
-    } catch (InterruptedException e) {
-      throw createIOException(fileName, inpMsg, errMsg, exitValue, e);
     } finally {
-      process.destroy();
+      IOUtils.closeStream(in);
-      if (in != null) in.close();
-      if (err != null) err.close();
     }
   }
   

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java

@@ -397,7 +397,7 @@ class win extends HardLinkCGWin {};
     //basic checks on array lengths
     assertEquals(5, win.hardLinkCommand.length); 
     assertEquals(7, win.hardLinkMultPrefix.length);
-    assertEquals(8, win.hardLinkMultSuffix.length);
+    assertEquals(7, win.hardLinkMultSuffix.length);
     assertEquals(4, win.getLinkCountCommand.length);
 
     assertTrue(win.hardLinkMultPrefix[4].equals("%f"));
@@ -406,9 +406,6 @@ class win extends HardLinkCGWin {};
     assertTrue(win.hardLinkMultDir.equals("\\%f"));
     //make sure "\\%f" was munged correctly
     assertEquals(3, ("\\%f").length()); 
-    assertTrue(win.hardLinkMultSuffix[7].equals("1>NUL"));
-    //make sure "1>NUL" was not munged
-    assertEquals(5, ("1>NUL").length()); 
     assertTrue(win.getLinkCountCommand[1].equals("hardlink"));
     //make sure "-c%h" was not munged
     assertEquals(4, ("-c%h").length());