big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9816. EntityGroupFSTimelineStore#scanActiveLogs fails when undesired files are present under /ats/active. Contribued by Prabhu Joseph.

Browse Source
This commit is contained in:
Abhishek Modi 2019-09-12 12:50:10 +05:30
parent 68612a0410
commit 44850f6784
2 changed files with 23 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
View File

@ -376,7 +376,12 @@ int scanActiveLogs(Path dir) throws IOException {
AppLogs logs = getAndSetActiveLog(appId, stat.getPath());
executor.execute(new ActiveLogParser(logs));
} else {
logsToScanCount += scanActiveLogs(stat.getPath());
if (stat.isDirectory()) {
logsToScanCount += scanActiveLogs(stat.getPath());
} else {
LOG.warn("Ignoring unexpected file in active directory {}",
stat.getPath());
}
}
}
return logsToScanCount;

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
View File

@ -510,6 +510,23 @@ public void testGetEntityPluginRead() throws Exception {
}
}
@Test
public void testScanActiveLogsWithInvalidFile() throws Exception {
Path invalidFile = new Path(testActiveDirPath, "invalidfile");
try {
if (!fs.exists(invalidFile)) {
fs.createNewFile(invalidFile);
}
store.scanActiveLogs();
} catch (StackOverflowError error) {
Assert.fail("EntityLogScanner crashed with StackOverflowError");
} finally {
if (fs.exists(invalidFile)) {
fs.delete(invalidFile, false);
}
}
}
@Test
public void testScanActiveLogsAndMoveToDonePluginRead() throws Exception {
EntityGroupFSTimelineStore store = null;