MAPREDUCE-3165. Ensure logging options are set correctly for MR AM and tasks. Contributed by Todd Lipcon.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1185887 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -1669,6 +1669,9 @@ Release 0.23.0 - Unreleased
     MAPREDUCE-2762. Cleanup MR staging directory on completion. (mahadev via
     acmurthy) 
 
+    MAPREDUCE-3165. Ensure logging options are set correctly for MR AM and
+    tasks. (todd via acmurthy) 
+
 Release 0.22.0 - Unreleased
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java

@@ -28,6 +28,7 @@
 import org.apache.hadoop.mapred.TaskLog.LogName;
 import org.apache.hadoop.mapreduce.ID;
 import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -93,7 +94,9 @@ public static void setVMEnv(Map<String, String> environment,
             MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV)
         );
 
-    // Set logging level
+    // Set logging level in the environment.
+    // This is so that, if the child forks another "bin/hadoop" (common in
+    // streaming) it will have the correct loglevel.
     environment.put(
         "HADOOP_ROOT_LOGGER", 
         getChildLogLevel(conf, task.isMapTask()) + ",CLA"); 
@@ -110,7 +113,7 @@ public static void setVMEnv(Map<String, String> environment,
     // properties.
     long logSize = TaskLog.getTaskLogLength(conf);
     Vector<String> logProps = new Vector<String>(4);
-    setupLog4jProperties(logProps, logSize);
+    setupLog4jProperties(task, logProps, logSize);
     Iterator<String> it = logProps.iterator();
     StringBuffer buffer = new StringBuffer();
     while (it.hasNext()) {
@@ -163,11 +166,11 @@ private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) {
     return adminClasspath + " " + userClasspath;
   }
 
-  private static void setupLog4jProperties(Vector<String> vargs,
+  private static void setupLog4jProperties(Task task,
+      Vector<String> vargs,
       long logSize) {
-    vargs.add("-Dlog4j.configuration=container-log4j.properties");
+    String logLevel = getChildLogLevel(task.conf, task.isMapTask()); 
-    vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
-    vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize);
   }
 
   public static List<String> getVMCommand(
@@ -222,7 +225,7 @@ public static List<String> getVMCommand(
 
     // Setup the log4j prop
     long logSize = TaskLog.getTaskLogLength(conf);
-    setupLog4jProperties(vargs, logSize);
+    setupLog4jProperties(task, vargs, logSize);
 
     if (conf.getProfileEnabled()) {
       if (conf.getProfileTaskRange(task.isMapTask()

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java

@@ -44,6 +44,7 @@
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.yarn.ContainerLogAppender;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -340,4 +341,19 @@ private static long[] getFileSizes(Configuration conf, String key) {
     }
     return result;
   }
+  
+  /**
+   * Add the JVM system properties necessary to configure {@link ContainerLogAppender}.
+   * @param logLevel the desired log level (eg INFO/WARN/DEBUG)
+   * @param logSize See {@link ContainerLogAppender#setTotalLogFileSize(long)}
+   * @param vargs the argument list to append to
+   */
+  public static void addLog4jSystemProperties(
+      String logLevel, long logSize, List<String> vargs) {
+    vargs.add("-Dlog4j.configuration=container-log4j.properties");
+    vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize);
+    vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA"); 
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -326,9 +326,9 @@ public interface MRJobConfig {
   public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1536m";
 
   /** Root Logging level passed to the MR app master.*/
-  public static final String MR_AM_LOG_OPTS = 
+  public static final String MR_AM_LOG_LEVEL = 
-    MR_AM_PREFIX+"log-opts";
+    MR_AM_PREFIX+"log.level";
-  public static final String DEFAULT_MR_AM_LOG_OPTS = "INFO";
+  public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO";
 
   /**The number of splits when reporting progress in MR*/
   public static final String MR_AM_NUM_PROGRESS_SPLITS = 
@@ -439,8 +439,9 @@ public interface MRJobConfig {
 
   public static final String HADOOP_WORK_DIR = "HADOOP_WORK_DIR";
 
+  // Environment variables used by Pipes. (TODO: these
+  // do not appear to be used by current pipes source code!)
   public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV";
-
   public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV";
 
   // This should be the directory where splits file gets localized on the node

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java

@@ -20,7 +20,9 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Vector;
 
@@ -320,14 +322,14 @@ public ApplicationSubmissionContext createApplicationSubmissionContext(
     }
 
     // Setup the command to run the AM
-    Vector<CharSequence> vargs = new Vector<CharSequence>(8);
+    List<String> vargs = new ArrayList<String>(8);
     vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
 
+    // TODO: why do we use 'conf' some places and 'jobConf' others?
     long logSize = TaskLog.getTaskLogLength(new JobConf(conf));
-    vargs.add("-Dlog4j.configuration=container-log4j.properties");
+    String logLevel = jobConf.get(
-    vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "="
+        MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
-        + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
-    vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize);
 
     vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
         MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));