From a2205a3b7279229b9ec94d0cc2796da8f40fd241 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Mon, 3 Jun 2013 02:35:34 +0000 Subject: [PATCH] YARN-720 and MAPREDUCE-5291. container-log4j.properties should not refer to mapreduce properties. Update MRApp to use YARN properties for log setup. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1488829 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../v2/app/job/impl/TestMapReduceChildJVM.java | 4 ++-- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 5 +++-- .../java/org/apache/hadoop/mapred/TaskLog.java | 4 ++-- .../org/apache/hadoop/mapreduce/MRJobConfig.java | 5 ----- .../java/org/apache/hadoop/mapred/TestTaskLog.java | 14 +++++++++----- .../hadoop/streaming/TestStreamingTaskLog.java | 4 ++-- hadoop-yarn-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/conf/YarnConfiguration.java | 8 ++++++++ .../src/main/resources/container-log4j.properties | 8 ++++---- 10 files changed, 36 insertions(+), 22 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b4e0c03418..9ae7fdac75 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -502,6 +502,9 @@ Release 2.1.0-beta - UNRELEASED MAPREDUCE-5177. Use common utils FileUtil#setReadable/Writable/Executable & FileUtil#canRead/Write/Execute. (Ivan Mitic via suresh) + MAPREDUCE-5291. Change MR App to use updated property names in + container-log4j.properties. (Zhijie Shen via sseth) + Release 2.0.5-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java index 0c4f38cf0c..26a2b95cef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java @@ -52,8 +52,8 @@ public void testCommandLine() throws Exception { " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir=" + envVar("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties" + - " -Dyarn.app.mapreduce.container.log.dir=" + - " -Dyarn.app.mapreduce.container.log.filesize=0" + + " -Dyarn.app.container.log.dir=" + + " -Dyarn.app.container.log.filesize=0" + " -Dhadoop.root.logger=INFO,CLA" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 3f9faa045b..fef0902c8d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -450,9 +450,10 @@ private static long[] getFileSizes(Configuration conf, String key) { public static void addLog4jSystemProperties( String logLevel, long logSize, List vargs) { vargs.add("-Dlog4j.configuration=container-log4j.properties"); - vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" + + vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); - vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize); + vargs.add( + "-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=" + logSize); vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA"); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index 082d2172d8..d929813b37 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -42,9 +42,9 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.util.ProcessTree; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.log4j.Appender; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -70,7 +70,7 @@ public class TaskLog { static LocalFileSystem localFS = null; public static String getMRv2LogDir() { - return System.getProperty(MRJobConfig.TASK_LOG_DIR); + return System.getProperty(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR); } public static File getTaskLogFile(TaskAttemptID taskid, boolean isCleanup, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 3aa33370e5..6b90796d0c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -599,11 +599,6 @@ public interface MRJobConfig { // Containers. public static final String APPLICATION_TOKENS_FILE = "appTokens"; - /** The log directory for the containers */ - public static final String TASK_LOG_DIR = MR_PREFIX + "container.log.dir"; - - public static final String TASK_LOG_SIZE = MR_PREFIX + "container.log.filesize"; - public static final String MAPREDUCE_V2_CHILD_CLASS = "org.apache.hadoop.mapred.YarnChild"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java index 328fc64048..0e16ab3d6f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java @@ -17,6 +17,11 @@ */ package org.apache.hadoop.mapred; +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -24,11 +29,9 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.mapred.TaskLog.LogName; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.Test; -import static junit.framework.Assert.*; -import static org.mockito.Mockito.*; - /** * TestCounters checks the sanity and recoverability of Queue */ @@ -42,7 +45,8 @@ public class TestTaskLog { @Test (timeout=50000) public void testTaskLog() throws IOException { // test TaskLog - System.setProperty(MRJobConfig.TASK_LOG_DIR, "testString"); + System.setProperty( + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR, "testString"); assertEquals(TaskLog.getMRv2LogDir(), "testString"); TaskAttemptID taid = mock(TaskAttemptID.class); JobID jid = new JobID("job", 1); @@ -115,7 +119,7 @@ public String readTaskLog(TaskLog.LogName filter, @Test (timeout=50000) public void testTaskLogWithoutTaskLogDir() throws IOException { // TaskLog tasklog= new TaskLog(); - System.clearProperty(MRJobConfig.TASK_LOG_DIR); + System.clearProperty(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR); // test TaskLog diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java index 11c3b4e9b0..20ef270617 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java @@ -155,8 +155,8 @@ private void runStreamJobAndValidateEnv() throws IOException { String env = MapReduceTestUtil.readOutput(outputPath, mr.createJobConf()); long logSize = USERLOG_LIMIT_KB * 1024; assertTrue("environment set for child is wrong", env.contains("INFO,CLA") - && env.contains("-Dyarn.app.mapreduce.container.log.dir=") - && env.contains("-Dyarn.app.mapreduce.container.log.filesize=" + logSize) + && env.contains("-Dyarn.app.container.log.dir=") + && env.contains("-Dyarn.app.container.log.filesize=" + logSize) && env.contains("-Dlog4j.configuration=")); } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8da28a4e39..d1af0b8449 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -84,6 +84,9 @@ Release 2.1.0-beta - UNRELEASED YARN-749. Rename ResourceRequest.(get,set)HostName to ResourceRequest.(get,set)ResourceName. (acmurthy) + YARN-720. container-log4j.properties should not refer to mapreduce + property names. (Zhijie Shen via sseth) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 46940ce263..fa1e74094d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -695,6 +695,14 @@ public class YarnConfiguration extends Configuration { */ public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false; + + /** The log directory for the containers */ + public static final String YARN_APP_CONTAINER_LOG_DIR = + YARN_PREFIX + "app.container.log.dir"; + + public static final String YARN_APP_CONTAINER_LOG_SIZE = + YARN_PREFIX + "app.container.log.filesize"; + //////////////////////////////// // Other Configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties index bc120f14a5..26fd1aa00e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties @@ -25,12 +25,12 @@ log4j.threshold=ALL # #Default values -yarn.app.mapreduce.container.log.dir=null -yarn.app.mapreduce.container.log.filesize=100 +yarn.app.container.log.dir=null +yarn.app.container.log.filesize=100 log4j.appender.CLA=org.apache.hadoop.yarn.ContainerLogAppender -log4j.appender.CLA.containerLogDir=${yarn.app.mapreduce.container.log.dir} -log4j.appender.CLA.totalLogFileSize=${yarn.app.mapreduce.container.log.filesize} +log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir} +log4j.appender.CLA.totalLogFileSize=${yarn.app.container.log.filesize} log4j.appender.CLA.layout=org.apache.log4j.PatternLayout log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n