big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11877. FileJournalManager#getLogFile should ignore in progress edit logs during JN sync. Contributed by Hanisha Koneru.

Browse Source
This commit is contained in:
Arpit Agarwal 2017-05-24 16:09:00 -07:00
parent 1c8dd6d3d1
commit 0e83ed5e73
4 changed files with 30 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
View File

@ -276,7 +276,7 @@ public URL buildURLToFetchLogs(long segmentTxId) {
try {
String path = GetJournalEditServlet.buildPath(
journalId, segmentTxId, nsInfo);
journalId, segmentTxId, nsInfo, true);
return new URL(httpServerURL, path);
} catch (MalformedURLException e) {
// should never get here.

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
View File

@ -72,6 +72,7 @@ public class GetJournalEditServlet extends HttpServlet {
static final String STORAGEINFO_PARAM = "storageInfo";
static final String JOURNAL_ID_PARAM = "jid";
static final String SEGMENT_TXID_PARAM = "segmentTxId";
static final String IN_PROGRESS_OK = "inProgressOk";
protected boolean isValidRequestor(HttpServletRequest request, Configuration conf)
throws IOException {
@ -186,6 +187,14 @@ public void doGet(final HttpServletRequest request,
final Configuration conf = (Configuration) getServletContext()
.getAttribute(JspHelper.CURRENT_CONF);
final String journalId = request.getParameter(JOURNAL_ID_PARAM);
final String inProgressOkStr = request.getParameter(IN_PROGRESS_OK);
final boolean inProgressOk;
if (inProgressOkStr != null &&
inProgressOkStr.equalsIgnoreCase("false")) {
inProgressOk = false;
} else {
inProgressOk = true;
}
QuorumJournalManager.checkJournalId(journalId);
final JNStorage storage = JournalNodeHttpServer
.getJournalFromContext(context, journalId).getStorage();
@ -210,8 +219,7 @@ public void doGet(final HttpServletRequest request,
// Synchronize on the FJM so that the file doesn't get finalized
// out from underneath us while we're in the process of opening
// it up.
EditLogFile elf = fjm.getLogFile(
segmentTxId);
EditLogFile elf = fjm.getLogFile(segmentTxId, inProgressOk);
if (elf == null) {
response.sendError(HttpServletResponse.SC_NOT_FOUND,
"No edit log found starting at txid " + segmentTxId);
@ -239,7 +247,7 @@ public void doGet(final HttpServletRequest request,
}
public static String buildPath(String journalId, long segmentTxId,
NamespaceInfo nsInfo) {
NamespaceInfo nsInfo, boolean inProgressOk) {
StringBuilder path = new StringBuilder("/getJournal?");
try {
path.append(JOURNAL_ID_PARAM).append("=")
@ -248,6 +256,8 @@ public static String buildPath(String journalId, long segmentTxId,
.append(segmentTxId);
path.append("&" + STORAGEINFO_PARAM).append("=")
.append(URLEncoder.encode(nsInfo.toColonSeparatedString(), "UTF-8"));
path.append("&" + IN_PROGRESS_OK).append("=")
.append(inProgressOk);
} catch (UnsupportedEncodingException e) {
// Never get here -- everyone supports UTF-8
throw new RuntimeException(e);

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
View File

@ -296,7 +296,7 @@ private void getMissingLogSegments(List<RemoteEditLog> thisJournalEditLogs,
}
String urlPath = GetJournalEditServlet.buildPath(jid, missingLog
.getStartTxId(), nsInfo);
.getStartTxId(), nsInfo, false);
url = new URL(remoteJNproxy.httpServerUrl, urlPath);
success = downloadMissingLogSegment(url, missingLog);
} catch (MalformedURLException e) {

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
View File

@ -451,16 +451,28 @@ public List<EditLogFile> getLogFiles(long fromTxId) throws IOException {
}
public EditLogFile getLogFile(long startTxId) throws IOException {
return getLogFile(sd.getCurrentDir(), startTxId);
return getLogFile(sd.getCurrentDir(), startTxId, true);
}
public EditLogFile getLogFile(long startTxId, boolean inProgressOk)
throws IOException {
return getLogFile(sd.getCurrentDir(), startTxId, inProgressOk);
}
public static EditLogFile getLogFile(File dir, long startTxId)
throws IOException {
return getLogFile(dir, startTxId, true);
}
public static EditLogFile getLogFile(File dir, long startTxId,
boolean inProgressOk) throws IOException {
List<EditLogFile> files = matchEditLogs(dir);
List<EditLogFile> ret = Lists.newLinkedList();
for (EditLogFile elf : files) {
if (elf.getFirstTxId() == startTxId) {
ret.add(elf);
if (inProgressOk || !elf.isInProgress()) {
ret.add(elf);
}
}
}