diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt index 918fd8b43f..de41bfe35e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt @@ -38,3 +38,5 @@ HDFS-3845. Fixes for edge cases in QJM recovery protocol (todd) HDFS-3877. QJM: Provide defaults for dfs.journalnode.*address (eli) HDFS-3863. Track last "committed" txid in QJM (todd) + +HDFS-3869. Expose non-file journal manager details in web UI (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java index 6b5e3f33ec..19206e6dbc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java @@ -140,4 +140,10 @@ public ListenableFuture acceptRecovery(SegmentStateProto log, * after this point, and any in-flight RPCs may throw an exception. */ public void close(); + + /** + * Append an HTML-formatted report for this logger's status to the provided + * StringBuilder. This is displayed on the NN web UI. + */ + public void appendHtmlReport(StringBuilder sb); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java index 96300db5fd..d158c445df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.ipc.RemoteException; +import org.apache.jasper.compiler.JspUtil; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -190,6 +191,24 @@ String getMajorityString() { int size() { return loggers.size(); } + + /** + * Append an HTML-formatted status readout on the current + * state of the underlying loggers. + * @param sb the StringBuilder to append to + */ + void appendHtmlReport(StringBuilder sb) { + sb.append(""); + sb.append("\n"); + for (AsyncLogger l : loggers) { + sb.append(""); + sb.append(""); + sb.append("\n"); + } + sb.append("
JNStatus
" + JspUtil.escapeXml(l.toString()) + ""); + l.appendHtmlReport(sb); + sb.append("
"); + } /** * @return the (mutable) list of loggers, for use in tests to diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index 80228f8900..8681513a8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -84,7 +84,12 @@ public class IPCLoggerChannel implements AsyncLogger { * The number of bytes of edits data still in the queue. */ private int queuedEditsSizeBytes = 0; - + + /** + * The highest txid that has been successfully logged on the remote JN. + */ + private long highestAckedTxId = 0; + /** * The maximum number of bytes that can be pending in the queue. * This keeps the writer from hitting OOME if one of the loggers @@ -262,6 +267,9 @@ public ListenableFuture sendEdits( public Void call() throws IOException { getProxy().journal(createReqInfo(), segmentTxId, firstTxnId, numTxns, data); + synchronized (IPCLoggerChannel.this) { + highestAckedTxId = firstTxnId + numTxns - 1; + } return null; } }); @@ -398,4 +406,14 @@ public Void call() throws IOException { public String toString() { return "Channel to journal node " + addr; } -} + + @Override + public synchronized void appendHtmlReport(StringBuilder sb) { + sb.append("Written txid ").append(highestAckedTxId); + long behind = committedTxId - highestAckedTxId; + assert behind >= 0; + if (behind > 0) { + sb.append(" (" + behind + " behind)"); + } + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java index e46676fd7a..4ea95ee2a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java @@ -109,4 +109,13 @@ protected void flushAndSync() throws IOException { loggers.setCommittedTxId(firstTxToFlush + numReadyTxns - 1); } } + + @Override + public String generateHtmlReport() { + StringBuilder sb = new StringBuilder(); + sb.append("Writing segment beginning at txid " + segmentTxId + "
\n"); + loggers.appendHtmlReport(sb); + return sb.toString(); + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java index cc9b62ccaf..ec418f532e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.jasper.compiler.JspUtil; /** * A generic abstract class to support journaling of edits logs into @@ -132,4 +133,12 @@ long getTotalSyncTime() { protected long getNumSync() { return numSync; } + + /** + * @return a short HTML snippet suitable for describing the current + * status of the stream + */ + public String generateHtmlReport() { + return JspUtil.escapeXml(this.toString()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java index dc2b15863c..0ba938d6e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.PriorityQueue; import java.util.SortedSet; +import java.util.concurrent.CopyOnWriteArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -147,7 +148,7 @@ JournalManager getManager() { return journal; } - private boolean isDisabled() { + boolean isDisabled() { return disabled; } @@ -165,8 +166,12 @@ public boolean isRequired() { return required; } } - - private List journals = Lists.newArrayList(); + + // COW implementation is necessary since some users (eg the web ui) call + // getAllJournalStreams() and then iterate. Since this is rarely + // mutated, there is no performance concern. + private List journals = + new CopyOnWriteArrayList(); final int minimumRedundantJournals; JournalSet(int minimumRedundantResources) { @@ -519,7 +524,6 @@ public void apply(JournalAndStream jas) throws IOException { } } - @VisibleForTesting List getAllJournalStreams() { return journals; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java index fa1a285158..04682c151b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; +import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.io.Text; @@ -61,6 +62,8 @@ import org.apache.hadoop.util.VersionInfo; import org.znerd.xmlenc.XMLOutputter; +import com.google.common.base.Preconditions; + class NamenodeJspHelper { static String getSafeModeText(FSNamesystem fsn) { if (!fsn.isInSafeMode()) @@ -213,6 +216,52 @@ void generateConfReport(JspWriter out, NameNode nn, out.print("\n"); } + + /** + * Generate an HTML report containing the current status of the HDFS + * journals. + */ + void generateJournalReport(JspWriter out, NameNode nn, + HttpServletRequest request) throws IOException { + FSEditLog log = nn.getFSImage().getEditLog(); + Preconditions.checkArgument(log != null, "no edit log set in %s", nn); + + out.println("

" + nn.getRole() + " Journal Status:

"); + + out.println("Current transaction ID: " + + nn.getFSImage().getLastAppliedOrWrittenTxId() + "
"); + + + boolean openForWrite = log.isOpenForWrite(); + + out.println("
"); + out.println("\n" + + ""); + for (JournalAndStream jas : log.getJournals()) { + out.print(""); + out.print(""); + } + + out.println("
Journal ManagerState
" + jas.getManager()); + if (jas.isRequired()) { + out.print(" [required]"); + } + out.print(""); + + if (jas.isDisabled()) { + out.print("Failed"); + } else if (openForWrite) { + EditLogOutputStream elos = jas.getCurrentStream(); + if (elos != null) { + out.println(elos.generateHtmlReport()); + } else { + out.println("not currently writing"); + } + } else { + out.println("open for read"); + } + out.println("
"); + } void generateHealthReport(JspWriter out, NameNode nn, HttpServletRequest request) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp index 275fd78c51..637d152b47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp @@ -60,8 +60,10 @@ <%= NamenodeJspHelper.getCorruptFilesWarning(fsn)%> <% healthjsp.generateHealthReport(out, nn, request); %> -
+<% healthjsp.generateJournalReport(out, nn, request); %> +
<% healthjsp.generateConfReport(out, nn, request); %> +
<% out.println(ServletUtil.htmlFooter()); %> diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java index 82f8dc1728..523008a6ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java @@ -17,17 +17,18 @@ */ package org.apache.hadoop.hdfs.qjournal; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import java.io.File; import java.io.IOException; +import java.net.URL; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -185,4 +186,41 @@ public void testMismatchedNNIsRejected() throws Exception { "Unable to start log segment 1: too few journals", ioe); } } + + @Test + public void testWebPageHasQjmInfo() throws Exception { + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + MiniDFSCluster.getBaseDirectory() + "/TestNNWithQJM/image"); + conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, + mjc.getQuorumJournalURI("myjournal").toString()); + + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0) + .manageNameDfsDirs(false) + .build(); + try { + URL url = new URL("http://localhost:" + + NameNode.getHttpAddress(cluster.getConfiguration(0)).getPort() + + "/dfshealth.jsp"); + + cluster.getFileSystem().mkdirs(TEST_PATH); + + String contents = DFSTestUtil.urlGet(url); + assertTrue(contents.contains("Channel to journal node")); + assertTrue(contents.contains("Written txid 2")); + + // Stop one JN, do another txn, and make sure it shows as behind + // stuck behind the others. + mjc.getJournalNode(0).stopAndJoin(0); + + cluster.getFileSystem().delete(TEST_PATH, true); + + contents = DFSTestUtil.urlGet(url); + System.out.println(contents); + assertTrue(contents.contains("(1 behind)")); + } finally { + cluster.shutdown(); + } + + } }