diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index da7fc14cb6..fed4a3bab6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -124,11 +124,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator { private final long rollingMonitorInterval; private final boolean logAggregationInRolling; private final NodeId nodeId; - // This variable is only for testing - private final AtomicBoolean waiting = new AtomicBoolean(false); - // This variable is only for testing + // These variables are only for testing + private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; + private int cleanupOldLogTimes = 0; private boolean renameTemporaryLogFileFailed = false; @@ -365,8 +365,9 @@ private void uploadLogsForContainers(boolean appFinished) { // Before upload logs, make sure the number of existing logs // is smaller than the configured NM log aggregation retention size. - if (uploadedLogsInThisCycle) { + if (uploadedLogsInThisCycle && logAggregationInRolling) { cleanOldLogs(); + cleanupOldLogTimes++; } if (writer != null) { @@ -689,4 +690,9 @@ public UserGroupInformation getUgi() { public int getLogAggregationTimes() { return this.logAggregationTimes; } + + @VisibleForTesting + int getCleanupOldLogTimes() { + return this.cleanupOldLogTimes; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 0392b3899d..0445c7917d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -2276,7 +2276,7 @@ public void testSkipUnnecessaryNNOperationsForShortJob() throws Exception { Records.newRecord(LogAggregationContext.class); logAggregationContext.setLogAggregationPolicyClassName( FailedOrKilledContainerLogAggregationPolicy.class.getName()); - verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2); + verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2, 0); } @Test (timeout = 20000) @@ -2290,13 +2290,13 @@ public void testSkipUnnecessaryNNOperationsForService() throws Exception { AMOnlyLogAggregationPolicy.class.getName()); contextWithAMOnly.setRolledLogsIncludePattern("sys*"); contextWithAMOnly.setRolledLogsExcludePattern("std_final"); - verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4); + verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4, 1); } private void verifySkipUnnecessaryNNOperations( LogAggregationContext logAggregationContext, - int expectedLogAggregationTimes, int expectedAggregationReportNum) - throws Exception { + int expectedLogAggregationTimes, int expectedAggregationReportNum, + int expectedCleanupOldLogsTimes) throws Exception { LogAggregationService logAggregationService = new LogAggregationService( dispatcher, this.context, this.delSrvc, super.dirsHandler); logAggregationService.init(this.conf); @@ -2307,7 +2307,7 @@ private void verifySkipUnnecessaryNNOperations( null, this.acls, logAggregationContext)); // Container finishes - String[] logFiles = new String[] { "stdout" }; + String[] logFiles = new String[] { "sysout" }; finishContainer(appId, logAggregationService, ContainerType.APPLICATION_MASTER, 1, 0, logFiles); AppLogAggregatorImpl aggregator = @@ -2327,6 +2327,8 @@ private void verifySkipUnnecessaryNNOperations( aggregator.getLogAggregationTimes()); assertEquals(expectedAggregationReportNum, this.context.getLogAggregationStatusForApps().size()); + assertEquals(expectedCleanupOldLogsTimes, + aggregator.getCleanupOldLogTimes()); } private int numOfLogsAvailable(LogAggregationService logAggregationService,