From fd57ab2002f97dcc83d455a5e0c770c8efde77a4 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Wed, 4 Feb 2015 17:36:28 -0800 Subject: [PATCH] MAPREDUCE-6059. Speed up history server startup time (Siqi Li via aw) --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a633b4ecb9..0fbf9a8f97 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -94,6 +94,8 @@ Trunk (Unreleased) MAPREDUCE-6013. [post-HADOOP-9902] mapred version is missing (Akira AJISAKA via aw) + MAPREDUCE-6059. Speed up history server startup time (Siqi Li via aw) + BUG FIXES MAPREDUCE-6191. Improve clearing stale state of Java serialization diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 40844df5f1..65f8a4febf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -263,6 +263,10 @@ public Collection values() { public HistoryFileInfo get(JobId jobId) { return cache.get(jobId); } + + public boolean isFull() { + return cache.size() >= maxSize; + } } /** @@ -668,6 +672,10 @@ void initExisting() throws IOException { for (FileStatus fs : timestampedDirList) { // TODO Could verify the correct format for these directories. addDirectoryToSerialNumberIndex(fs.getPath()); + } + for (int i= timestampedDirList.size() - 1; + i >= 0 && !jobListCache.isFull(); i--) { + FileStatus fs = timestampedDirList.get(i); addDirectoryToJobListCache(fs.getPath()); } }