YARN-4155. TestLogAggregationService.testLogAggregationServiceWithInterval failing. (Bibin A Chundatt via stevel)
This commit is contained in:
parent
bafeb6c7bc
commit
e2d59e2c7b
@ -957,6 +957,9 @@ Release 2.8.0 - UNRELEASED
|
||||
YARN-4000. RM crashes with NPE if leaf queue becomes parent queue during restart.
|
||||
(Varun Saxena via jianhe)
|
||||
|
||||
YARN-4155. TestLogAggregationService.testLogAggregationServiceWithInterval failing
|
||||
(Bibin A Chundatt via stevel)
|
||||
|
||||
Release 2.7.2 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
@ -22,6 +22,7 @@
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@ -248,7 +249,7 @@ protected DeletionService createDeletionService() {
|
||||
public void delete(String user, Path subDir, Path... baseDirs) {
|
||||
// Don't do any deletions.
|
||||
LOG.info("Psuedo delete: user - " + user + ", subDir - " + subDir
|
||||
+ ", baseDirs - " + baseDirs);
|
||||
+ ", baseDirs - " + Arrays.asList(baseDirs));
|
||||
};
|
||||
};
|
||||
}
|
||||
|
@ -788,7 +788,9 @@ private void writeContainerLogs(File appLogDir, ContainerId containerId,
|
||||
// ContainerLogDir should be created
|
||||
String containerStr = ConverterUtils.toString(containerId);
|
||||
File containerLogDir = new File(appLogDir, containerStr);
|
||||
containerLogDir.mkdir();
|
||||
boolean created = containerLogDir.mkdirs();
|
||||
LOG.info("Created Dir:" + containerLogDir.getAbsolutePath() + " status :"
|
||||
+ created);
|
||||
for (String fileType : fileName) {
|
||||
Writer writer11 = new FileWriter(new File(containerLogDir, fileType));
|
||||
writer11.write(containerStr + " Hello " + fileType + "!");
|
||||
@ -1926,7 +1928,7 @@ private void testLogAggregationService(boolean retentionSizeLimitation)
|
||||
//configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
|
||||
//have fully qualified path
|
||||
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
|
||||
"file://" + this.remoteRootLogDir.getAbsolutePath());
|
||||
this.remoteRootLogDir.toURI().toString());
|
||||
this.conf.setLong(
|
||||
YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
|
||||
3600);
|
||||
@ -1943,7 +1945,8 @@ private void testLogAggregationService(boolean retentionSizeLimitation)
|
||||
// again in next cycle.
|
||||
this.conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600);
|
||||
|
||||
ApplicationId application = BuilderUtils.newApplicationId(123456, 1);
|
||||
ApplicationId application =
|
||||
BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
|
||||
ApplicationAttemptId appAttemptId =
|
||||
BuilderUtils.newApplicationAttemptId(application, 1);
|
||||
ContainerId container = createContainer(appAttemptId, 1,
|
||||
@ -2009,8 +2012,10 @@ private void testLogAggregationService(boolean retentionSizeLimitation)
|
||||
|
||||
// Same logs will not be aggregated again.
|
||||
// Only one aggregated log file in Remote file directory.
|
||||
Assert.assertEquals(numOfLogsAvailable(logAggregationService,
|
||||
application, true, null), 1);
|
||||
Assert.assertTrue(
|
||||
"Only one aggregated log file in Remote file directory expected",
|
||||
waitAndCheckLogNum(logAggregationService, application, 50, 1, true,
|
||||
null));
|
||||
|
||||
Thread.sleep(2000);
|
||||
|
||||
@ -2134,6 +2139,7 @@ private int numOfLogsAvailable(LogAggregationService logAggregationService,
|
||||
FileContext.getFileContext(qualifiedLogDir.toUri(), this.conf)
|
||||
.listStatus(appLogDir);
|
||||
} catch (FileNotFoundException fnf) {
|
||||
LOG.info("Context file not vailable: " + fnf);
|
||||
return -1;
|
||||
}
|
||||
int count = 0;
|
||||
@ -2143,13 +2149,17 @@ private int numOfLogsAvailable(LogAggregationService logAggregationService,
|
||||
if (filename.contains(LogAggregationUtils.TMP_FILE_SUFFIX)
|
||||
|| (lastLogFile != null && filename.contains(lastLogFile)
|
||||
&& sizeLimited)) {
|
||||
LOG.info("fileName :" + filename);
|
||||
LOG.info("lastLogFile :" + lastLogFile);
|
||||
return -1;
|
||||
}
|
||||
if (filename.contains(LogAggregationUtils
|
||||
.getNodeString(logAggregationService.getNodeId()))) {
|
||||
LOG.info("Node list filename :" + filename);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
LOG.info("File Count :" + count);
|
||||
return count;
|
||||
}
|
||||
|
||||
@ -2158,13 +2168,17 @@ private boolean waitAndCheckLogNum(
|
||||
int maxAttempts, int expectNum, boolean sizeLimited, String lastLogFile)
|
||||
throws IOException, InterruptedException {
|
||||
int count = 0;
|
||||
while (numOfLogsAvailable(logAggregationService, application, sizeLimited,
|
||||
lastLogFile) != expectNum && count <= maxAttempts) {
|
||||
int logFiles=numOfLogsAvailable(logAggregationService, application, sizeLimited,
|
||||
lastLogFile);
|
||||
while ((logFiles != expectNum)
|
||||
&& (count <= maxAttempts)) {
|
||||
Thread.sleep(500);
|
||||
count++;
|
||||
logFiles =
|
||||
numOfLogsAvailable(logAggregationService, application, sizeLimited,
|
||||
lastLogFile);
|
||||
}
|
||||
return numOfLogsAvailable(logAggregationService, application, sizeLimited,
|
||||
lastLogFile) == expectNum;
|
||||
return (logFiles == expectNum);
|
||||
}
|
||||
|
||||
private static class LogFileStatusInLastCycle {
|
||||
|
Loading…
Reference in New Issue
Block a user