big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6436. JobHistory cache issue. Contributed by Kai Sasaki

Browse Source
This commit is contained in:
Zhihai Xu 2015-12-15 00:58:23 -08:00
parent 96d307e1e3
commit 5b7078d069
2 changed files with 36 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
View File

@ -219,13 +219,21 @@ public HistoryFileInfo addIfAbsent(HistoryFileInfo fileInfo) {
// keeping the cache size exactly at the maximum.
Iterator<JobId> keys = cache.navigableKeySet().iterator();
long cutoff = System.currentTimeMillis() - maxAge;
// MAPREDUCE-6436: In order to reduce the number of logs written
// in case of a lot of move pending histories.
JobId firstInIntermediateKey = null;
int inIntermediateCount = 0;
JobId firstMoveFailedKey = null;
int moveFailedCount = 0;
while(cache.size() > maxSize && keys.hasNext()) {
JobId key = keys.next();
HistoryFileInfo firstValue = cache.get(key);
if(firstValue != null) {
synchronized(firstValue) {
if (firstValue.isMovePending()) {
if(firstValue.didMoveFail() &&
if(firstValue.didMoveFail() &&
firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
cache.remove(key);
//Now lets try to delete it
@ -236,8 +244,17 @@ public HistoryFileInfo addIfAbsent(HistoryFileInfo fileInfo) {
" that could not be moved to done.", e);
}
} else {
LOG.warn("Waiting to remove " + key
+ " from JobListCache because it is not in done yet.");
if (firstValue.didMoveFail()) {
if (moveFailedCount == 0) {
firstMoveFailedKey = key;
}
moveFailedCount += 1;
} else {
if (inIntermediateCount == 0) {
firstInIntermediateKey = key;
}
inIntermediateCount += 1;
}
}
} else {
cache.remove(key);
@ -245,6 +262,20 @@ public HistoryFileInfo addIfAbsent(HistoryFileInfo fileInfo) {
}
}
}
// Log output only for first jobhisotry in pendings to restrict
// the total number of logs.
if (inIntermediateCount > 0) {
LOG.warn("Waiting to remove IN_INTERMEDIATE state histories " +
"(e.g. " + firstInIntermediateKey + ") from JobListCache " +
"because it is not in done yet. Total count is " +
inIntermediateCount + ".");
}
if (moveFailedCount > 0) {
LOG.warn("Waiting to remove MOVE_FAILED state histories " +
"(e.g. " + firstMoveFailedKey + ") from JobListCache " +
"because it is not in done yet. Total count is " +
moveFailedCount + ".");
}
}
return old;
}

2
hadoop-yarn-project/CHANGES.txt
View File

@ -2030,6 +2030,8 @@ Release 2.6.4 - UNRELEASED
IMPROVEMENTS
MAPREDUCE-6436. JobHistory cache issue. (Kai Sasaki via zxu)
OPTIMIZATIONS
BUG FIXES