diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 548d868a68..917d32ba58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1370,6 +1370,11 @@ public class YarnConfiguration extends Configuration {
public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX
+ "log-aggregation.retain-seconds";
public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1;
+
+ public static final String LOG_AGGREGATION_DEBUG_FILESIZE = YARN_PREFIX
+ + "log-aggregation.debug.filesize";
+ public static final long DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE
+ = 100 * 1024 * 1024;
/**
* How long to wait between aggregated log retention checks. If set to
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8582522b63..f379844cf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1291,6 +1291,14 @@
-1
+
+ The log files created under NM Local Directories
+ will be logged if it exceeds the configured bytes. This
+ only takes effect if log4j level is at least Debug.
+ yarn.log-aggregation.debug.filesize
+ 104857600
+
+
Specify which log file controllers we will support. The first
file controller we add will be used to write the aggregated logs.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index fdac2e439d..ef14d2aba8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -109,7 +109,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
private final AtomicBoolean waiting = new AtomicBoolean(false);
private int logAggregationTimes = 0;
private int cleanupOldLogTimes = 0;
-
+ private long logFileSizeThreshold;
private boolean renameTemporaryLogFileFailed = false;
private final Map containerLogAggregators =
@@ -176,6 +176,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
this.nodeId = nodeId;
this.logAggPolicy = getLogAggPolicy(conf);
this.recoveredLogInitedTime = recoveredLogInitedTime;
+ this.logFileSizeThreshold =
+ conf.getLong(YarnConfiguration.LOG_AGGREGATION_DEBUG_FILESIZE,
+ YarnConfiguration.DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE);
if (logAggregationFileController == null) {
// by default, use T-File Controller
this.logAggregationFileController = new LogAggregationTFileController();
@@ -330,6 +333,19 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
uploadedLogsInThisCycle = true;
List uploadedFilePathsInThisCycleList = new ArrayList<>();
uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle);
+ if (LOG.isDebugEnabled()) {
+ for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) {
+ try {
+ long fileSize = lfs.getFileStatus(uploadedFilePath).getLen();
+ if (fileSize >= logFileSizeThreshold) {
+ LOG.debug("Log File " + uploadedFilePath
+ + " size is " + fileSize + " bytes");
+ }
+ } catch (Exception e1) {
+ LOG.error("Failed to get log file size " + e1);
+ }
+ }
+ }
deletionTask = new FileDeletionTask(delService,
this.userUgi.getShortUserName(), null,
uploadedFilePathsInThisCycleList);