MAPREDUCE-6686. Add a way to download the job config from the mapred CLI (rkanter)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java

@@ -39,6 +39,8 @@
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.TIPStatus;
@@ -106,6 +108,7 @@ public int run(String[] argv) throws Exception {
     int fromEvent = 0;
     int nEvents = 0;
     int jpvalue = 0;
+    String configOutFile = null;
     boolean getStatus = false;
     boolean getCounter = false;
     boolean killJob = false;
@@ -121,6 +124,7 @@ public int run(String[] argv) throws Exception {
     boolean failTask = false;
     boolean setJobPriority = false;
     boolean logs = false;
+    boolean downloadConfig = false;
 
     if ("-submit".equals(cmd)) {
       if (argv.length != 2) {
@@ -295,6 +299,14 @@ public int run(String[] argv) throws Exception {
         displayUsage(cmd);
         return exitCode;
       }
+    } else if ("-config".equals(cmd)) {
+      downloadConfig = true;
+      if (argv.length != 3) {
+        displayUsage(cmd);
+        return exitCode;
+      }
+      jobid = argv[1];
+      configOutFile = argv[2];
     } else {
       displayUsage(cmd);
       return exitCode;
@@ -473,6 +485,22 @@ public int run(String[] argv) throws Exception {
             System.out.println(e.getMessage());
           }
         }
+      } else if (downloadConfig) {
+        Job job = getJob(JobID.forName(jobid));
+        if (job == null) {
+          System.out.println("Could not find job " + jobid);
+        } else {
+          String jobFile = job.getJobFile();
+          if (jobFile == null || jobFile.isEmpty()) {
+            System.out.println("Config file for job " + jobFile +
+                " could not be found.");
+          } else {
+            Path configPath = new Path(jobFile);
+            FileSystem fs = FileSystem.get(getConf());
+            fs.copyToLocalFile(configPath, new Path(configOutFile));
+            exitCode = 0;
+          }
+        }
       }
     } catch (RemoteException re) {
       IOException unwrappedException = re.unwrapRemoteException();
@@ -550,6 +578,8 @@ private void displayUsage(String cmd) {
       System.err.println(prefix + "[" + cmd +
           " <job-id> <task-attempt-id>]. " +
           " <task-attempt-id> is optional to get task attempt logs.");
+    } else if ("-config".equals(cmd)) {
+      System.err.println(prefix + "[" + cmd + " <job-id> <file>]");
     } else {
       System.err.printf(prefix + "<command> <args>%n");
       System.err.printf("\t[-submit <job-file>]%n");
@@ -571,7 +601,8 @@ private void displayUsage(String cmd) {
         "Valid values for <task-state> are " + taskStates);
       System.err.printf("\t[-kill-task <task-attempt-id>]%n");
       System.err.printf("\t[-fail-task <task-attempt-id>]%n");
-      System.err.printf("\t[-logs <job-id> <task-attempt-id>]%n%n");
+      System.err.printf("\t[-logs <job-id> <task-attempt-id>]%n");
+      System.err.printf("\t[-config <job-id> <file>%n%n");
       ToolRunner.printGenericCommandUsage(System.out);
     }
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md

@@ -81,7 +81,7 @@ Copy file or directories recursively. More information can be found at
 
 Command to interact with Map Reduce Jobs.
 
-Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobHistoryFile|jobId> [-outfile <file>] [-format <human|json>]] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>]`
+Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobHistoryFile|jobId> [-outfile <file>] [-format <human|json>]] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>] [-config <job-id> <file>]`
 
 | COMMAND\_OPTION | Description |
 |:---- |:---- |
@@ -99,6 +99,7 @@ Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id
 | -list-blacklisted-trackers | List the black listed task trackers in the cluster. This command is not supported in MRv2 based cluster. |
 | -list-attempt-ids *job-id* *task-type* *task-state* | List the attempt-ids based on the task type and the status given. Valid values for task-type are REDUCE, MAP. Valid values for task-state are running, pending, completed, failed, killed. |
 | -logs *job-id* *task-attempt-id* | Dump the container log for a job if taskAttemptId is not specified, otherwise dump the log for the task with the specified taskAttemptId. The logs will be dumped in system out. |
+| -config *job-id* *file* | Download the job configuration file. |
 
 ### `pipes`
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java

@@ -168,6 +168,8 @@ public void testJobClient() throws Exception {
     testfailTask(conf);
     // kill job
     testKillJob(conf);
+    // download job config
+    testConfig(jobId, conf);
   }
 
   /**
@@ -534,6 +536,32 @@ private void checkHistoryJSONFileOutput(String jobId,
     assertEquals(0, out.size());
   }
 
+  /**
+   * download job config
+   */
+  private void testConfig(String jobId, Configuration conf) throws Exception {
+    CLI jc = createJobClient();
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+    // bad arguments
+    int exitCode = runTool(conf, jc, new String[] { "-config" }, out);
+    assertEquals("Exit code", -1, exitCode);
+    exitCode = runTool(conf, jc, new String[] { "-config job_invalid foo.xml" },
+        out);
+    assertEquals("Exit code", -1, exitCode);
+
+    // good arguments
+    File outFile = File.createTempFile("config", ".xml");
+    exitCode = runTool(conf, jc, new String[] { "-config", jobId,
+        outFile.toString()}, out);
+    assertEquals("Exit code", 0, exitCode);
+    BufferedReader br = new BufferedReader(new FileReader(outFile));
+    String line = br.readLine();
+    br.close();
+    assertEquals("<?xml version=\"1.0\" encoding=\"UTF-8\" " +
+        "standalone=\"no\"?><configuration>", line);
+  }
+
   /**
    * print job events list 
    */