big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7131. Job History Server has race condition where it moves files from intermediate to finished but thinks file is in intermediate. Contributed by Anthony Hsu

Browse Source
This commit is contained in:
Jason Lowe 2018-09-06 14:13:29 -05:00
parent b6c543fe45
commit eb0b5a844f
2 changed files with 63 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
View File

@ -1089,7 +1089,17 @@ public HistoryFileInfo getFileInfo(JobId jobId) throws IOException {
private void moveToDoneNow(final Path src, final Path target)
throws IOException {
LOG.info("Moving " + src.toString() + " to " + target.toString());
try {
intermediateDoneDirFc.rename(src, target, Options.Rename.NONE);
} catch (FileNotFoundException e) {
if (doneDirFc.util().exists(target)) {
LOG.info("Source file " + src.toString() + " not found, but target "
+ "file " + target.toString() + " already exists. Move already "
+ "happened.");
} else {
throw e;
}
}
}
private String getJobSummary(FileContext fc, Path path) throws IOException {

52
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
View File

@ -26,6 +26,7 @@
import java.util.List;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@ -341,6 +342,57 @@ public void testHistoryFileInfoShouldReturnCompletedJobIfMaxNotConfiged()
}
/**
* This test sets up a scenario where the history files have already been
* moved to the "done" directory (so the "intermediate" directory is empty),
* but then moveToDone() is called again on the same history file. It
* validates that the second moveToDone() still succeeds rather than throws a
* FileNotFoundException.
*/
@Test
public void testMoveToDoneAlreadyMovedSucceeds() throws Exception {
HistoryFileManagerTest historyFileManager = new HistoryFileManagerTest();
long jobTimestamp = 1535436603000L;
String job = "job_" + jobTimestamp + "_123456789";
String intermediateDirectory = "/" + UUID.randomUUID();
String doneDirectory = "/" + UUID.randomUUID();
Configuration conf = dfsCluster.getConfiguration(0);
conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
intermediateDirectory);
conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, doneDirectory);
Path intermediateHistoryFilePath = new Path(intermediateDirectory + "/"
+ job + ".jhist");
Path intermediateConfFilePath = new Path(intermediateDirectory + "/"
+ job + "_conf.xml");
Path doneHistoryFilePath = new Path(doneDirectory + "/"
+ JobHistoryUtils.timestampDirectoryComponent(jobTimestamp) + "/123456/"
+ job + ".jhist");
Path doneConfFilePath = new Path(doneDirectory + "/"
+ JobHistoryUtils.timestampDirectoryComponent(jobTimestamp)
+ "/123456/" + job + "_conf.xml");
dfsCluster.getFileSystem().createNewFile(doneHistoryFilePath);
dfsCluster.getFileSystem().createNewFile(doneConfFilePath);
historyFileManager.serviceInit(conf);
JobIndexInfo jobIndexInfo = new JobIndexInfo();
jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(job)));
jobIndexInfo.setFinishTime(jobTimestamp);
HistoryFileInfo info = historyFileManager.getHistoryFileInfo(
intermediateHistoryFilePath, intermediateConfFilePath, null,
jobIndexInfo, false);
info.moveToDone();
Assert.assertFalse(info.isMovePending());
Assert.assertEquals(doneHistoryFilePath.toString(),
info.getHistoryFile().toUri().getPath());
Assert.assertEquals(doneConfFilePath.toString(),
info.getConfFile().toUri().getPath());
}
static class HistoryFileManagerTest extends HistoryFileManager {
public HistoryFileManagerTest() {
super();