From 512e39e89df4653bc36301048c4db05e39551ee3 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 20 Sep 2023 08:04:05 +0800 Subject: [PATCH] Revert "MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0. (#6042) Contributed by Chenyu Zheng." (#6103) This reverts commit ab2bc90e090564b7466ab1b759fbb849612ae82b. --- .../src/main/java/org/apache/hadoop/mapred/TaskLog.java | 3 +-- .../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 6 ++---- .../src/main/resources/mapred-default.xml | 7 ++++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index fdfa3e7710..a0223dedd6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -466,8 +466,7 @@ public static long getTaskLogLength(JobConf conf) { } public static long getTaskLogLimitBytes(Configuration conf) { - return conf.getLong(JobContext.TASK_USERLOG_LIMIT, JobContext.DEFAULT_TASK_USERLOG_LIMIT) * - 1024; + return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 24689e09da..8ec984e777 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -410,8 +410,6 @@ public interface MRJobConfig { public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb"; - public static final int DEFAULT_TASK_USERLOG_LIMIT = 10240; - public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent"; public static final String MAP_INPUT_FILE = "mapreduce.map.input.file"; @@ -760,11 +758,11 @@ public interface MRJobConfig { public static final String MR_AM_LOG_KB = MR_AM_PREFIX + "container.log.limit.kb"; - public static final int DEFAULT_MR_AM_LOG_KB = 10240; + public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll public static final String MR_AM_LOG_BACKUPS = MR_AM_PREFIX + "container.log.backups"; - public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll + public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; /**The number of splits when reporting progress in MR*/ public static final String MR_AM_NUM_PROGRESS_SPLITS = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 493e4c3a37..a6d68acda3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -823,15 +823,16 @@ mapreduce.task.userlog.limit.kb - 10240 - The maximum size of user-logs of each task in KB. + 0 + The maximum size of user-logs of each task in KB. 0 disables the cap. yarn.app.mapreduce.am.container.log.limit.kb - 10240 + 0 The maximum size of the MRAppMaster attempt container logs in KB. + 0 disables the cap.