From 0e83ed5e7372c801c9fee01df91b6b56de467ab1 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 24 May 2017 16:09:00 -0700 Subject: [PATCH] HDFS-11877. FileJournalManager#getLogFile should ignore in progress edit logs during JN sync. Contributed by Hanisha Koneru. --- .../hdfs/qjournal/client/IPCLoggerChannel.java | 2 +- .../qjournal/server/GetJournalEditServlet.java | 16 +++++++++++++--- .../qjournal/server/JournalNodeSyncer.java | 2 +- .../server/namenode/FileJournalManager.java | 18 +++++++++++++++--- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index cbfe5b8d48..6cd892c7ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -276,7 +276,7 @@ public URL buildURLToFetchLogs(long segmentTxId) { try { String path = GetJournalEditServlet.buildPath( - journalId, segmentTxId, nsInfo); + journalId, segmentTxId, nsInfo, true); return new URL(httpServerURL, path); } catch (MalformedURLException e) { // should never get here. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java index 2335ea55d2..e96fd4da60 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java @@ -72,6 +72,7 @@ public class GetJournalEditServlet extends HttpServlet { static final String STORAGEINFO_PARAM = "storageInfo"; static final String JOURNAL_ID_PARAM = "jid"; static final String SEGMENT_TXID_PARAM = "segmentTxId"; + static final String IN_PROGRESS_OK = "inProgressOk"; protected boolean isValidRequestor(HttpServletRequest request, Configuration conf) throws IOException { @@ -186,6 +187,14 @@ public void doGet(final HttpServletRequest request, final Configuration conf = (Configuration) getServletContext() .getAttribute(JspHelper.CURRENT_CONF); final String journalId = request.getParameter(JOURNAL_ID_PARAM); + final String inProgressOkStr = request.getParameter(IN_PROGRESS_OK); + final boolean inProgressOk; + if (inProgressOkStr != null && + inProgressOkStr.equalsIgnoreCase("false")) { + inProgressOk = false; + } else { + inProgressOk = true; + } QuorumJournalManager.checkJournalId(journalId); final JNStorage storage = JournalNodeHttpServer .getJournalFromContext(context, journalId).getStorage(); @@ -210,8 +219,7 @@ public void doGet(final HttpServletRequest request, // Synchronize on the FJM so that the file doesn't get finalized // out from underneath us while we're in the process of opening // it up. - EditLogFile elf = fjm.getLogFile( - segmentTxId); + EditLogFile elf = fjm.getLogFile(segmentTxId, inProgressOk); if (elf == null) { response.sendError(HttpServletResponse.SC_NOT_FOUND, "No edit log found starting at txid " + segmentTxId); @@ -239,7 +247,7 @@ public void doGet(final HttpServletRequest request, } public static String buildPath(String journalId, long segmentTxId, - NamespaceInfo nsInfo) { + NamespaceInfo nsInfo, boolean inProgressOk) { StringBuilder path = new StringBuilder("/getJournal?"); try { path.append(JOURNAL_ID_PARAM).append("=") @@ -248,6 +256,8 @@ public static String buildPath(String journalId, long segmentTxId, .append(segmentTxId); path.append("&" + STORAGEINFO_PARAM).append("=") .append(URLEncoder.encode(nsInfo.toColonSeparatedString(), "UTF-8")); + path.append("&" + IN_PROGRESS_OK).append("=") + .append(inProgressOk); } catch (UnsupportedEncodingException e) { // Never get here -- everyone supports UTF-8 throw new RuntimeException(e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java index 788c5dee51..73defc219d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java @@ -296,7 +296,7 @@ private void getMissingLogSegments(List thisJournalEditLogs, } String urlPath = GetJournalEditServlet.buildPath(jid, missingLog - .getStartTxId(), nsInfo); + .getStartTxId(), nsInfo, false); url = new URL(remoteJNproxy.httpServerUrl, urlPath); success = downloadMissingLogSegment(url, missingLog); } catch (MalformedURLException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index 5d628255a6..c71c09ad4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -451,16 +451,28 @@ public List getLogFiles(long fromTxId) throws IOException { } public EditLogFile getLogFile(long startTxId) throws IOException { - return getLogFile(sd.getCurrentDir(), startTxId); + return getLogFile(sd.getCurrentDir(), startTxId, true); } - + + public EditLogFile getLogFile(long startTxId, boolean inProgressOk) + throws IOException { + return getLogFile(sd.getCurrentDir(), startTxId, inProgressOk); + } + public static EditLogFile getLogFile(File dir, long startTxId) throws IOException { + return getLogFile(dir, startTxId, true); + } + + public static EditLogFile getLogFile(File dir, long startTxId, + boolean inProgressOk) throws IOException { List files = matchEditLogs(dir); List ret = Lists.newLinkedList(); for (EditLogFile elf : files) { if (elf.getFirstTxId() == startTxId) { - ret.add(elf); + if (inProgressOk || !elf.isInProgress()) { + ret.add(elf); + } } }