big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6221. Entities missing from ATS when summary log file info got returned to the ATS before the domain log. Contributed by Xiaomin Zhang

Browse Source
This commit is contained in:
Szilard Nemeth 2021-07-31 14:53:30 +02:00
parent a272adc5fa
commit 0a0628b1cb
2 changed files with 27 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
View File

@ -808,10 +808,11 @@ private void addSummaryLog(String attemptDirName,
LogInfo log;
if (isDomainLog) {
log = new DomainLogInfo(attemptDirName, filename, owner);
summaryLogs.add(0, log);
} else {
log = new EntityLogInfo(attemptDirName, filename, owner);
summaryLogs.add(log);
}
summaryLogs.add(log);
}
private synchronized void addDetailLog(String attemptDirName,

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
View File

@ -226,6 +226,31 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
}
}
@Test
public void testAppLogsDomainLogLastlyScanned() throws Exception {
EntityGroupFSTimelineStore.AppLogs appLogs =
store.new AppLogs(mainTestAppId, mainTestAppDirPath,
AppState.COMPLETED);
Path attemptDirPath = new Path(new Path(testActiveDirPath,
mainTestAppId.toString()),
getAttemptDirName(mainTestAppId));
//Delete the domain log from AppDirPath so first scan won't find it
fs.delete(new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME), false);
appLogs.scanForLogs();
List<LogInfo> summaryLogs = appLogs.getSummaryLogs();
assertEquals(1, summaryLogs.size());
assertEquals(TEST_SUMMARY_LOG_FILE_NAME, summaryLogs.get(0).getFilename());
//Generate the domain log
FSDataOutputStream out = fs.create(
new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME));
out.close();
appLogs.scanForLogs();
assertEquals(2, summaryLogs.size());
assertEquals(TEST_DOMAIN_LOG_FILE_NAME, summaryLogs.get(0).getFilename());
}
@Test
public void testMoveToDone() throws Exception {
EntityGroupFSTimelineStore.AppLogs appLogs =