big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-720 and MAPREDUCE-5291. container-log4j.properties should not refer to mapreduce properties. Update MRApp to use YARN properties for log setup. Contributed by Zhijie Shen.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1488829 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Siddharth Seth 2013-06-03 02:35:34 +00:00
parent f27e484636
commit a2205a3b72
10 changed files with 36 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -502,6 +502,9 @@ Release 2.1.0-beta - UNRELEASED
MAPREDUCE-5177. Use common utils FileUtil#setReadable/Writable/Executable &
FileUtil#canRead/Write/Execute. (Ivan Mitic via suresh)
MAPREDUCE-5291. Change MR App to use updated property names in
container-log4j.properties. (Zhijie Shen via sseth)
Release 2.0.5-alpha - UNRELEASED
INCOMPATIBLE CHANGES

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
View File

@ -52,8 +52,8 @@ public void testCommandLine() throws Exception {
" -Dhadoop.metrics.log.level=WARN" +
" -Xmx200m -Djava.io.tmpdir=" + envVar("PWD") + "/tmp" +
" -Dlog4j.configuration=container-log4j.properties" +
" -Dyarn.app.mapreduce.container.log.dir=<LOG_DIR>" +
" -Dyarn.app.mapreduce.container.log.filesize=0" +
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
" -Dyarn.app.container.log.filesize=0" +
" -Dhadoop.root.logger=INFO,CLA" +
" org.apache.hadoop.mapred.YarnChild 127.0.0.1" +
" 54321" +

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
View File

@ -450,9 +450,10 @@ private static long[] getFileSizes(Configuration conf, String key) {
public static void addLog4jSystemProperties(
String logLevel, long logSize, List<String> vargs) {
vargs.add("-Dlog4j.configuration=container-log4j.properties");
vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" +
vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
ApplicationConstants.LOG_DIR_EXPANSION_VAR);
vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize);
vargs.add(
"-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=" + logSize);
vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
}

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java
View File

@ -42,9 +42,9 @@
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.util.ProcessTree;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.log4j.Appender;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@ -70,7 +70,7 @@ public class TaskLog {
static LocalFileSystem localFS = null;
public static String getMRv2LogDir() {
return System.getProperty(MRJobConfig.TASK_LOG_DIR);
return System.getProperty(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR);
}
public static File getTaskLogFile(TaskAttemptID taskid, boolean isCleanup,

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -599,11 +599,6 @@ public interface MRJobConfig {
// Containers.
public static final String APPLICATION_TOKENS_FILE = "appTokens";
/** The log directory for the containers */
public static final String TASK_LOG_DIR = MR_PREFIX + "container.log.dir";
public static final String TASK_LOG_SIZE = MR_PREFIX + "container.log.filesize";
public static final String MAPREDUCE_V2_CHILD_CLASS =
"org.apache.hadoop.mapred.YarnChild";

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java
View File

@ -17,6 +17,11 @@
*/
package org.apache.hadoop.mapred;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@ -24,11 +29,9 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.mapred.TaskLog.LogName;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Test;
import static junit.framework.Assert.*;
import static org.mockito.Mockito.*;
/**
* TestCounters checks the sanity and recoverability of Queue
*/
@ -42,7 +45,8 @@ public class TestTaskLog {
@Test (timeout=50000)
public void testTaskLog() throws IOException {
// test TaskLog
System.setProperty(MRJobConfig.TASK_LOG_DIR, "testString");
System.setProperty(
YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR, "testString");
assertEquals(TaskLog.getMRv2LogDir(), "testString");
TaskAttemptID taid = mock(TaskAttemptID.class);
JobID jid = new JobID("job", 1);
@ -115,7 +119,7 @@ public String readTaskLog(TaskLog.LogName filter,
@Test (timeout=50000)
public void testTaskLogWithoutTaskLogDir() throws IOException {
// TaskLog tasklog= new TaskLog();
System.clearProperty(MRJobConfig.TASK_LOG_DIR);
System.clearProperty(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR);
// test TaskLog

4
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
View File

@ -155,8 +155,8 @@ private void runStreamJobAndValidateEnv() throws IOException {
String env = MapReduceTestUtil.readOutput(outputPath, mr.createJobConf());
long logSize = USERLOG_LIMIT_KB * 1024;
assertTrue("environment set for child is wrong", env.contains("INFO,CLA")
&& env.contains("-Dyarn.app.mapreduce.container.log.dir=")
&& env.contains("-Dyarn.app.mapreduce.container.log.filesize=" + logSize)
&& env.contains("-Dyarn.app.container.log.dir=")
&& env.contains("-Dyarn.app.container.log.filesize=" + logSize)
&& env.contains("-Dlog4j.configuration="));
}
}

3
hadoop-yarn-project/CHANGES.txt
View File

@ -84,6 +84,9 @@ Release 2.1.0-beta - UNRELEASED
YARN-749. Rename ResourceRequest.(get,set)HostName to
ResourceRequest.(get,set)ResourceName. (acmurthy)
YARN-720. container-log4j.properties should not refer to mapreduce
property names. (Zhijie Shen via sseth)
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -695,6 +695,14 @@ public class YarnConfiguration extends Configuration {
*/
public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
/** The log directory for the containers */
public static final String YARN_APP_CONTAINER_LOG_DIR =
YARN_PREFIX + "app.container.log.dir";
public static final String YARN_APP_CONTAINER_LOG_SIZE =
YARN_PREFIX + "app.container.log.filesize";
////////////////////////////////
// Other Configs
////////////////////////////////

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties
View File

@ -25,12 +25,12 @@ log4j.threshold=ALL
#
#Default values
yarn.app.mapreduce.container.log.dir=null
yarn.app.mapreduce.container.log.filesize=100
yarn.app.container.log.dir=null
yarn.app.container.log.filesize=100
log4j.appender.CLA=org.apache.hadoop.yarn.ContainerLogAppender
log4j.appender.CLA.containerLogDir=${yarn.app.mapreduce.container.log.dir}
log4j.appender.CLA.totalLogFileSize=${yarn.app.mapreduce.container.log.filesize}
log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir}
log4j.appender.CLA.totalLogFileSize=${yarn.app.container.log.filesize}
log4j.appender.CLA.layout=org.apache.log4j.PatternLayout
log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n