From ab787f44aabfff0cd01b79a08a52ffaf923558b3 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Tue, 18 Oct 2011 21:45:38 +0000 Subject: [PATCH] MAPREDUCE-3165. Ensure logging options are set correctly for MR AM and tasks. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1185887 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../apache/hadoop/mapred/MapReduceChildJVM.java | 17 ++++++++++------- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 16 ++++++++++++++++ .../apache/hadoop/mapreduce/MRJobConfig.java | 9 +++++---- .../org/apache/hadoop/mapred/YARNRunner.java | 12 +++++++----- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index fd1804c341..f7640a90b4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1669,6 +1669,9 @@ Release 0.23.0 - Unreleased MAPREDUCE-2762. Cleanup MR staging directory on completion. (mahadev via acmurthy) + MAPREDUCE-3165. Ensure logging options are set correctly for MR AM and + tasks. (todd via acmurthy) + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java index 8c80617ae6..914606a24e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java @@ -28,6 +28,7 @@ import org.apache.hadoop.mapred.TaskLog.LogName; import org.apache.hadoop.mapreduce.ID; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; @@ -93,7 +94,9 @@ public static void setVMEnv(Map environment, MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV) ); - // Set logging level + // Set logging level in the environment. + // This is so that, if the child forks another "bin/hadoop" (common in + // streaming) it will have the correct loglevel. environment.put( "HADOOP_ROOT_LOGGER", getChildLogLevel(conf, task.isMapTask()) + ",CLA"); @@ -110,7 +113,7 @@ public static void setVMEnv(Map environment, // properties. long logSize = TaskLog.getTaskLogLength(conf); Vector logProps = new Vector(4); - setupLog4jProperties(logProps, logSize); + setupLog4jProperties(task, logProps, logSize); Iterator it = logProps.iterator(); StringBuffer buffer = new StringBuffer(); while (it.hasNext()) { @@ -163,11 +166,11 @@ private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) { return adminClasspath + " " + userClasspath; } - private static void setupLog4jProperties(Vector vargs, + private static void setupLog4jProperties(Task task, + Vector vargs, long logSize) { - vargs.add("-Dlog4j.configuration=container-log4j.properties"); - vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); - vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize); + String logLevel = getChildLogLevel(task.conf, task.isMapTask()); + MRApps.addLog4jSystemProperties(logLevel, logSize, vargs); } public static List getVMCommand( @@ -222,7 +225,7 @@ public static List getVMCommand( // Setup the log4j prop long logSize = TaskLog.getTaskLogLength(conf); - setupLog4jProperties(vargs, logSize); + setupLog4jProperties(task, vargs, logSize); if (conf.getProfileEnabled()) { if (conf.getProfileTaskRange(task.isMapTask() diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 0eee5ce8e4..2aa26da639 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; +import org.apache.hadoop.yarn.ContainerLogAppender; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -340,4 +341,19 @@ private static long[] getFileSizes(Configuration conf, String key) { } return result; } + + /** + * Add the JVM system properties necessary to configure {@link ContainerLogAppender}. + * @param logLevel the desired log level (eg INFO/WARN/DEBUG) + * @param logSize See {@link ContainerLogAppender#setTotalLogFileSize(long)} + * @param vargs the argument list to append to + */ + public static void addLog4jSystemProperties( + String logLevel, long logSize, List vargs) { + vargs.add("-Dlog4j.configuration=container-log4j.properties"); + vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" + + ApplicationConstants.LOG_DIR_EXPANSION_VAR); + vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize); + vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA"); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 41b3897257..38669cdeb0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -326,9 +326,9 @@ public interface MRJobConfig { public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1536m"; /** Root Logging level passed to the MR app master.*/ - public static final String MR_AM_LOG_OPTS = - MR_AM_PREFIX+"log-opts"; - public static final String DEFAULT_MR_AM_LOG_OPTS = "INFO"; + public static final String MR_AM_LOG_LEVEL = + MR_AM_PREFIX+"log.level"; + public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO"; /**The number of splits when reporting progress in MR*/ public static final String MR_AM_NUM_PROGRESS_SPLITS = @@ -439,8 +439,9 @@ public interface MRJobConfig { public static final String HADOOP_WORK_DIR = "HADOOP_WORK_DIR"; + // Environment variables used by Pipes. (TODO: these + // do not appear to be used by current pipes source code!) public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV"; - public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV"; // This should be the directory where splits file gets localized on the node diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index aceb02378b..527720a4bd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -20,7 +20,9 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Vector; @@ -320,14 +322,14 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( } // Setup the command to run the AM - Vector vargs = new Vector(8); + List vargs = new ArrayList(8); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); + // TODO: why do we use 'conf' some places and 'jobConf' others? long logSize = TaskLog.getTaskLogLength(new JobConf(conf)); - vargs.add("-Dlog4j.configuration=container-log4j.properties"); - vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" - + ApplicationConstants.LOG_DIR_EXPANSION_VAR); - vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize); + String logLevel = jobConf.get( + MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL); + MRApps.addLog4jSystemProperties(logLevel, logSize, vargs); vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));