big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by Xuan Gong.

Browse Source
This commit is contained in:
Zhijie Shen 2014-10-26 12:57:31 -07:00
parent 9ada9284d8
commit caecd9fffe
3 changed files with 18 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -760,6 +760,9 @@ Release 2.6.0 - UNRELEASED
YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port. YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port.
(Naganarasimha G R via xgong) (Naganarasimha G R via xgong)
YARN-2734. Skipped sub-folders in the local log dir when aggregating logs.
(Xuan Gong via zjshen)
Release 2.5.1 - 2014-09-05 Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
View File

@ -213,6 +213,12 @@ public void write(DataOutputStream out, Set<File> pendingUploadFiles)
Collections.sort(fileList); Collections.sort(fileList);
for (File logFile : fileList) { for (File logFile : fileList) {
// We only aggregate top level files.
// Ignore anything inside sub-folders.
if (logFile.isDirectory()) {
LOG.warn(logFile.getAbsolutePath() + " is a directory. Ignore it.");
continue;
}
FileInputStream in = null; FileInputStream in = null;
try { try {

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
View File

@ -194,6 +194,14 @@ public void testReadAcontainerLogs1() throws Exception {
int numChars = 80000; int numChars = 80000;
// create a sub-folder under srcFilePath
// and create file logs in this sub-folder.
// We only aggregate top level files.
// So, this log file should be ignored.
Path subDir = new Path(srcFilePath, "subDir");
fs.mkdirs(subDir);
writeSrcFile(subDir, "logs", numChars);
// create file stderr and stdout in containerLogDir // create file stderr and stdout in containerLogDir
writeSrcFile(srcFilePath, "stderr", numChars); writeSrcFile(srcFilePath, "stderr", numChars);
writeSrcFile(srcFilePath, "stdout", numChars); writeSrcFile(srcFilePath, "stdout", numChars);
@ -238,6 +246,7 @@ public void testReadAcontainerLogs1() throws Exception {
+ "\nLog Contents:\n".length() + numChars; + "\nLog Contents:\n".length() + numChars;
Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars)); Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
Assert.assertTrue("Log Contents not matched", s.contains("Log Contents")); Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));