big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0. (#6042) Contributed by Chenyu Zheng.

Browse Source 
Reviewed-by: Shilun Fan <slfan1989@apache.org>
Signed-off-by: Shilun Fan <slfan1989@apache.org>
This commit is contained in:
zhengchenyu 2023-09-15 20:50:11 +08:00 committed by GitHub
parent e283375cdf
commit ab2bc90e09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java
View File

@ -466,7 +466,8 @@ public static long getTaskLogLength(JobConf conf) {
} }
public static long getTaskLogLimitBytes(Configuration conf) { public static long getTaskLogLimitBytes(Configuration conf) {
return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024; return conf.getLong(JobContext.TASK_USERLOG_LIMIT, JobContext.DEFAULT_TASK_USERLOG_LIMIT) *
1024;
} }

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -410,6 +410,8 @@ public interface MRJobConfig {
public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb"; public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb";
public static final int DEFAULT_TASK_USERLOG_LIMIT = 10240;
public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent"; public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent";
public static final String MAP_INPUT_FILE = "mapreduce.map.input.file"; public static final String MAP_INPUT_FILE = "mapreduce.map.input.file";
@ -758,11 +760,11 @@ public interface MRJobConfig {
public static final String MR_AM_LOG_KB = public static final String MR_AM_LOG_KB =
MR_AM_PREFIX + "container.log.limit.kb"; MR_AM_PREFIX + "container.log.limit.kb";
public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll public static final int DEFAULT_MR_AM_LOG_KB = 10240;
public static final String MR_AM_LOG_BACKUPS = public static final String MR_AM_LOG_BACKUPS =
MR_AM_PREFIX + "container.log.backups"; MR_AM_PREFIX + "container.log.backups";
public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll
/**The number of splits when reporting progress in MR*/ /**The number of splits when reporting progress in MR*/
public static final String MR_AM_NUM_PROGRESS_SPLITS = public static final String MR_AM_NUM_PROGRESS_SPLITS =

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
View File

@ -823,16 +823,15 @@
<property> <property>
<name>mapreduce.task.userlog.limit.kb</name> <name>mapreduce.task.userlog.limit.kb</name>
<value>0</value> <value>10240</value>
<description>The maximum size of user-logs of each task in KB. 0 disables the cap. <description>The maximum size of user-logs of each task in KB.
</description> </description>
</property> </property>
<property> <property>
<name>yarn.app.mapreduce.am.container.log.limit.kb</name> <name>yarn.app.mapreduce.am.container.log.limit.kb</name>
<value>0</value> <value>10240</value>
<description>The maximum size of the MRAppMaster attempt container logs in KB. <description>The maximum size of the MRAppMaster attempt container logs in KB.
0 disables the cap.
</description> </description>
</property> </property>