From e2d59e2c7b18a3d2f3917558463905a6b087b16f Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Sun, 18 Oct 2015 12:03:46 +0100 Subject: [PATCH] YARN-4155. TestLogAggregationService.testLogAggregationServiceWithInterval failing. (Bibin A Chundatt via stevel) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../BaseContainerManagerTest.java | 3 +- .../TestLogAggregationService.java | 32 +++++++++++++------ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8aebe21a4d..a328f82f0b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -957,6 +957,9 @@ Release 2.8.0 - UNRELEASED YARN-4000. RM crashes with NPE if leaf queue becomes parent queue during restart. (Varun Saxena via jianhe) + YARN-4155. TestLogAggregationService.testLogAggregationServiceWithInterval failing + (Bibin A Chundatt via stevel) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index 532944bf5b..42574946f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -248,7 +249,7 @@ protected DeletionService createDeletionService() { public void delete(String user, Path subDir, Path... baseDirs) { // Don't do any deletions. LOG.info("Psuedo delete: user - " + user + ", subDir - " + subDir - + ", baseDirs - " + baseDirs); + + ", baseDirs - " + Arrays.asList(baseDirs)); }; }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 0b33634450..101fef093e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -788,7 +788,9 @@ private void writeContainerLogs(File appLogDir, ContainerId containerId, // ContainerLogDir should be created String containerStr = ConverterUtils.toString(containerId); File containerLogDir = new File(appLogDir, containerStr); - containerLogDir.mkdir(); + boolean created = containerLogDir.mkdirs(); + LOG.info("Created Dir:" + containerLogDir.getAbsolutePath() + " status :" + + created); for (String fileType : fileName) { Writer writer11 = new FileWriter(new File(containerLogDir, fileType)); writer11.write(containerStr + " Hello " + fileType + "!"); @@ -1926,7 +1928,7 @@ private void testLogAggregationService(boolean retentionSizeLimitation) //configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to //have fully qualified path this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - "file://" + this.remoteRootLogDir.getAbsolutePath()); + this.remoteRootLogDir.toURI().toString()); this.conf.setLong( YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600); @@ -1943,7 +1945,8 @@ private void testLogAggregationService(boolean retentionSizeLimitation) // again in next cycle. this.conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600); - ApplicationId application = BuilderUtils.newApplicationId(123456, 1); + ApplicationId application = + BuilderUtils.newApplicationId(System.currentTimeMillis(), 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application, 1); ContainerId container = createContainer(appAttemptId, 1, @@ -2009,8 +2012,10 @@ private void testLogAggregationService(boolean retentionSizeLimitation) // Same logs will not be aggregated again. // Only one aggregated log file in Remote file directory. - Assert.assertEquals(numOfLogsAvailable(logAggregationService, - application, true, null), 1); + Assert.assertTrue( + "Only one aggregated log file in Remote file directory expected", + waitAndCheckLogNum(logAggregationService, application, 50, 1, true, + null)); Thread.sleep(2000); @@ -2134,6 +2139,7 @@ private int numOfLogsAvailable(LogAggregationService logAggregationService, FileContext.getFileContext(qualifiedLogDir.toUri(), this.conf) .listStatus(appLogDir); } catch (FileNotFoundException fnf) { + LOG.info("Context file not vailable: " + fnf); return -1; } int count = 0; @@ -2143,13 +2149,17 @@ private int numOfLogsAvailable(LogAggregationService logAggregationService, if (filename.contains(LogAggregationUtils.TMP_FILE_SUFFIX) || (lastLogFile != null && filename.contains(lastLogFile) && sizeLimited)) { + LOG.info("fileName :" + filename); + LOG.info("lastLogFile :" + lastLogFile); return -1; } if (filename.contains(LogAggregationUtils .getNodeString(logAggregationService.getNodeId()))) { + LOG.info("Node list filename :" + filename); count++; } } + LOG.info("File Count :" + count); return count; } @@ -2158,13 +2168,17 @@ private boolean waitAndCheckLogNum( int maxAttempts, int expectNum, boolean sizeLimited, String lastLogFile) throws IOException, InterruptedException { int count = 0; - while (numOfLogsAvailable(logAggregationService, application, sizeLimited, - lastLogFile) != expectNum && count <= maxAttempts) { + int logFiles=numOfLogsAvailable(logAggregationService, application, sizeLimited, + lastLogFile); + while ((logFiles != expectNum) + && (count <= maxAttempts)) { Thread.sleep(500); count++; + logFiles = + numOfLogsAvailable(logAggregationService, application, sizeLimited, + lastLogFile); } - return numOfLogsAvailable(logAggregationService, application, sizeLimited, - lastLogFile) == expectNum; + return (logFiles == expectNum); } private static class LogFileStatusInLastCycle {