diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index 1cadf84bf6..9e964e1381 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -61,8 +61,13 @@ public class JHAdminConfig {
MR_HISTORY_PREFIX + "cleaner.interval-ms";
public static final long DEFAULT_MR_HISTORY_CLEANER_INTERVAL_MS =
1 * 24 * 60 * 60 * 1000l; //1 day
-
-
+
+ /** Always scan user dir, irrespective of dir modification time.*/
+ public static final String MR_HISTORY_ALWAYS_SCAN_USER_DIR =
+ MR_HISTORY_PREFIX + "always-scan-user-dir";
+ public static final boolean DEFAULT_MR_HISTORY_ALWAYS_SCAN_USER_DIR =
+ false;
+
/** The number of threads to handle client API requests.*/
public static final String MR_HISTORY_CLIENT_THREAD_COUNT =
MR_HISTORY_PREFIX + "client.thread-count";
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index dcb312ce3b..9f33d6553c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1774,6 +1774,15 @@
+
+ mapreduce.jobhistory.always-scan-user-dir
+ false
+ Some Cloud FileSystems do not currently update the
+ modification time of directories. To support these filesystems, this
+ configuration value should be set to 'true'.
+
+
+
mapreduce.jobhistory.done-dir
${yarn.app.mapreduce.am.staging-dir}/history/done
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index a07ca26c1c..7fe99a28b9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -324,7 +324,13 @@ public synchronized void scanIfNeeded(FileStatus fs) {
// so we need to have additional check.
// Note: modTime (X second Y millisecond) could be casted to X second or
// X+1 second.
- if (modTime != newModTime
+ // MAPREDUCE-7101: Some Cloud FileSystems do not currently update the
+ // modification time of directories. For these, we scan every time if
+ // the 'alwaysScan' is true.
+ boolean alwaysScan = conf.getBoolean(
+ JHAdminConfig.MR_HISTORY_ALWAYS_SCAN_USER_DIR,
+ JHAdminConfig.DEFAULT_MR_HISTORY_ALWAYS_SCAN_USER_DIR);
+ if (alwaysScan || modTime != newModTime
|| (scanTime/1000) == (modTime/1000)
|| (scanTime/1000 + 1) == (modTime/1000)) {
// reset scanTime before scanning happens