big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6252. JobHistoryServer should not fail when encountering a

Browse Source 
missing directory. Contributed by Craig Welch.
This commit is contained in:
Devaraj K 2015-04-27 15:01:42 +05:30
parent 618ba707f0
commit 5e67c4d384
3 changed files with 37 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -349,6 +349,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are
broken due to MAPREDUCE-6297. (Siqi Li via gera) broken due to MAPREDUCE-6297. (Siqi Li via gera)
MAPREDUCE-6252. JobHistoryServer should not fail when encountering a
missing directory. (Craig Welch via devaraj)
Release 2.7.1 - UNRELEASED Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

19
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
View File

@ -740,17 +740,22 @@ private void addDirectoryToJobListCache(Path path) throws IOException {
} }
} }
private static List<FileStatus> scanDirectory(Path path, FileContext fc, @VisibleForTesting
protected static List<FileStatus> scanDirectory(Path path, FileContext fc,
PathFilter pathFilter) throws IOException { PathFilter pathFilter) throws IOException {
path = fc.makeQualified(path); path = fc.makeQualified(path);
List<FileStatus> jhStatusList = new ArrayList<FileStatus>(); List<FileStatus> jhStatusList = new ArrayList<FileStatus>();
RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path); try {
while (fileStatusIter.hasNext()) { RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path);
FileStatus fileStatus = fileStatusIter.next(); while (fileStatusIter.hasNext()) {
Path filePath = fileStatus.getPath(); FileStatus fileStatus = fileStatusIter.next();
if (fileStatus.isFile() && pathFilter.accept(filePath)) { Path filePath = fileStatus.getPath();
jhStatusList.add(fileStatus); if (fileStatus.isFile() && pathFilter.accept(filePath)) {
jhStatusList.add(fileStatus);
}
} }
} catch (FileNotFoundException fe) {
LOG.error("Error while scanning directory " + path, fe);
} }
return jhStatusList; return jhStatusList;
} }

22
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
View File

@ -21,13 +21,17 @@
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileNotFoundException;
import java.util.UUID; import java.util.UUID;
import java.util.List;
import org.junit.Assert; import org.junit.Assert;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@ -45,6 +49,8 @@
import org.junit.Test; import org.junit.Test;
import org.junit.rules.TestName; import org.junit.rules.TestName;
import static org.mockito.Mockito.*;
public class TestHistoryFileManager { public class TestHistoryFileManager {
private static MiniDFSCluster dfsCluster = null; private static MiniDFSCluster dfsCluster = null;
private static MiniDFSCluster dfsCluster2 = null; private static MiniDFSCluster dfsCluster2 = null;
@ -199,4 +205,20 @@ public void run() {
testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock); testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock);
} }
@Test
public void testScanDirectory() throws Exception {
Path p = new Path("any");
FileContext fc = mock(FileContext.class);
when(fc.makeQualified(p)).thenReturn(p);
when(fc.listStatus(p)).thenThrow(new FileNotFoundException());
List<FileStatus> lfs = HistoryFileManager.scanDirectory(p, fc, null);
//primarily, succcess is that an exception was not thrown. Also nice to
//check this
Assert.assertNotNull(lfs);
}
} }