From 5e67c4d384193b38a85655c8f93193596821faa5 Mon Sep 17 00:00:00 2001
From: Devaraj K <devaraj@apache.org>
Date: Mon, 27 Apr 2015 15:01:42 +0530
Subject: [PATCH] MAPREDUCE-6252. JobHistoryServer should not fail when
 encountering a missing directory. Contributed by Craig Welch.

---
 hadoop-mapreduce-project/CHANGES.txt          |  3 +++
 .../mapreduce/v2/hs/HistoryFileManager.java   | 19 ++++++++++------
 .../v2/hs/TestHistoryFileManager.java         | 22 +++++++++++++++++++
 3 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 4166e6aa95..f9488fbb33 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -349,6 +349,9 @@ Release 2.8.0 - UNRELEASED
     MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are
     broken due to MAPREDUCE-6297. (Siqi Li via gera)
 
+    MAPREDUCE-6252. JobHistoryServer should not fail when encountering a 
+    missing directory. (Craig Welch via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 65f8a4febf..69f814dc5f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -740,17 +740,22 @@ private void addDirectoryToJobListCache(Path path) throws IOException {
     }
   }
 
-  private static List<FileStatus> scanDirectory(Path path, FileContext fc,
+  @VisibleForTesting
+  protected static List<FileStatus> scanDirectory(Path path, FileContext fc,
       PathFilter pathFilter) throws IOException {
     path = fc.makeQualified(path);
     List<FileStatus> jhStatusList = new ArrayList<FileStatus>();
-    RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path);
-    while (fileStatusIter.hasNext()) {
-      FileStatus fileStatus = fileStatusIter.next();
-      Path filePath = fileStatus.getPath();
-      if (fileStatus.isFile() && pathFilter.accept(filePath)) {
-        jhStatusList.add(fileStatus);
+    try {
+      RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path);
+      while (fileStatusIter.hasNext()) {
+        FileStatus fileStatus = fileStatusIter.next();
+        Path filePath = fileStatus.getPath();
+        if (fileStatus.isFile() && pathFilter.accept(filePath)) {
+          jhStatusList.add(fileStatus);
+        }
       }
+    } catch (FileNotFoundException fe) {
+      LOG.error("Error while scanning directory " + path, fe);
     }
     return jhStatusList;
   }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index e2e943a3d6..1c5cc5ce74 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
@@ -21,13 +21,17 @@
 
 import java.io.File;
 import java.io.FileOutputStream;
+import java.io.FileNotFoundException;
 import java.util.UUID;
+import java.util.List;
 
 import org.junit.Assert;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -45,6 +49,8 @@
 import org.junit.Test;
 import org.junit.rules.TestName;
 
+import static org.mockito.Mockito.*;
+
 public class TestHistoryFileManager {
   private static MiniDFSCluster dfsCluster = null;
   private static MiniDFSCluster dfsCluster2 = null;
@@ -199,4 +205,20 @@ public void run() {
     testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock);
   }
 
+  @Test
+  public void testScanDirectory() throws Exception {
+
+    Path p = new Path("any");
+    FileContext fc = mock(FileContext.class);
+    when(fc.makeQualified(p)).thenReturn(p);
+    when(fc.listStatus(p)).thenThrow(new FileNotFoundException());
+
+    List<FileStatus> lfs = HistoryFileManager.scanDirectory(p, fc, null);
+
+    //primarily, succcess is that an exception was not thrown.  Also nice to
+    //check this
+    Assert.assertNotNull(lfs);
+
+  }
+
 }