diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 548d868a68..917d32ba58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1370,6 +1370,11 @@ public class YarnConfiguration extends Configuration { public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX + "log-aggregation.retain-seconds"; public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; + + public static final String LOG_AGGREGATION_DEBUG_FILESIZE = YARN_PREFIX + + "log-aggregation.debug.filesize"; + public static final long DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE + = 100 * 1024 * 1024; /** * How long to wait between aggregated log retention checks. If set to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 8582522b63..f379844cf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1291,6 +1291,14 @@ -1 + + The log files created under NM Local Directories + will be logged if it exceeds the configured bytes. This + only takes effect if log4j level is at least Debug. + yarn.log-aggregation.debug.filesize + 104857600 + + Specify which log file controllers we will support. The first file controller we add will be used to write the aggregated logs. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index fdac2e439d..ef14d2aba8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -109,7 +109,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator { private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; private int cleanupOldLogTimes = 0; - + private long logFileSizeThreshold; private boolean renameTemporaryLogFileFailed = false; private final Map containerLogAggregators = @@ -176,6 +176,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator { this.nodeId = nodeId; this.logAggPolicy = getLogAggPolicy(conf); this.recoveredLogInitedTime = recoveredLogInitedTime; + this.logFileSizeThreshold = + conf.getLong(YarnConfiguration.LOG_AGGREGATION_DEBUG_FILESIZE, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE); if (logAggregationFileController == null) { // by default, use T-File Controller this.logAggregationFileController = new LogAggregationTFileController(); @@ -330,6 +333,19 @@ public class AppLogAggregatorImpl implements AppLogAggregator { uploadedLogsInThisCycle = true; List uploadedFilePathsInThisCycleList = new ArrayList<>(); uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); + if (LOG.isDebugEnabled()) { + for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) { + try { + long fileSize = lfs.getFileStatus(uploadedFilePath).getLen(); + if (fileSize >= logFileSizeThreshold) { + LOG.debug("Log File " + uploadedFilePath + + " size is " + fileSize + " bytes"); + } + } catch (Exception e1) { + LOG.error("Failed to get log file size " + e1); + } + } + } deletionTask = new FileDeletionTask(delService, this.userUgi.getShortUserName(), null, uploadedFilePathsInThisCycleList);