HDFS-15970. Print network topology on the web (#2896)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java

@@ -166,7 +166,7 @@ void start() throws IOException {
 
     httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
     httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
-    setupServlets(httpServer, conf);
+    setupServlets(httpServer);
     httpServer.start();
 
     int connIdx = 0;
@@ -243,7 +243,7 @@ void setAliasMap(InMemoryAliasMap aliasMap) {
     httpServer.setAttribute(ALIASMAP_ATTRIBUTE_KEY, aliasMap);
   }
 
-  private static void setupServlets(HttpServer2 httpServer, Configuration conf) {
+  private static void setupServlets(HttpServer2 httpServer) {
     httpServer.addInternalServlet("startupProgress",
         StartupProgressServlet.PATH_SPEC, StartupProgressServlet.class);
     httpServer.addInternalServlet("fsck", "/fsck", FsckServlet.class,
@@ -253,6 +253,8 @@ private static void setupServlets(HttpServer2 httpServer, Configuration conf) {
     httpServer.addInternalServlet(IsNameNodeActiveServlet.SERVLET_NAME,
         IsNameNodeActiveServlet.PATH_SPEC,
         IsNameNodeActiveServlet.class);
+    httpServer.addInternalServlet("topology",
+        NetworkTopologyServlet.PATH_SPEC, NetworkTopologyServlet.class);
   }
 
   static FSImage getFsImageFromContext(ServletContext context) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NetworkTopologyServlet.java

@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.Node;
+import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.util.StringUtils;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.HttpHeaders;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
+/**
+ * A servlet to print out the network topology.
+ */
+@InterfaceAudience.Private
+public class NetworkTopologyServlet extends DfsServlet {
+
+  public static final String PATH_SPEC = "/topology";
+
+  protected static final String FORMAT_JSON = "json";
+  protected static final String FORMAT_TEXT = "text";
+
+  @Override
+  public void doGet(HttpServletRequest request, HttpServletResponse response)
+      throws IOException {
+    final ServletContext context = getServletContext();
+
+    String format = parseAcceptHeader(request);
+    if (FORMAT_TEXT.equals(format)) {
+      response.setContentType("text/plain; charset=UTF-8");
+    } else if (FORMAT_JSON.equals(format)) {
+      response.setContentType("application/json; charset=UTF-8");
+    }
+
+    NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context);
+    BlockManager bm = nn.getNamesystem().getBlockManager();
+    List<Node> leaves = bm.getDatanodeManager().getNetworkTopology()
+        .getLeaves(NodeBase.ROOT);
+
+    try (PrintStream out = new PrintStream(
+            response.getOutputStream(), false, "UTF-8")) {
+      printTopology(out, leaves, format);
+    } catch (Throwable t) {
+      String errMsg = "Print network topology failed. "
+              + StringUtils.stringifyException(t);
+      response.sendError(HttpServletResponse.SC_GONE, errMsg);
+      throw new IOException(errMsg);
+    } finally {
+      response.getOutputStream().close();
+    }
+  }
+
+  /**
+   * Display each rack and the nodes assigned to that rack, as determined
+   * by the NameNode, in a hierarchical manner.  The nodes and racks are
+   * sorted alphabetically.
+   *
+   * @param stream print stream
+   * @param leaves leaves nodes under base scope
+   * @param format the response format
+   */
+  public void printTopology(PrintStream stream, List<Node> leaves,
+      String format) throws BadFormatException, IOException {
+    if (leaves.isEmpty()) {
+      stream.print("No DataNodes");
+      return;
+    }
+
+    // Build a map of rack -> nodes
+    Map<String, TreeSet<String>> tree = new HashMap<>();
+    for(Node dni : leaves) {
+      String location = dni.getNetworkLocation();
+      String name = dni.getName();
+
+      tree.putIfAbsent(location, new TreeSet<>());
+      tree.get(location).add(name);
+    }
+
+    // Sort the racks (and nodes) alphabetically, display in order
+    ArrayList<String> racks = new ArrayList<>(tree.keySet());
+    Collections.sort(racks);
+
+    if (FORMAT_JSON.equals(format)) {
+      printJsonFormat(stream, tree, racks);
+    } else if (FORMAT_TEXT.equals(format)) {
+      printTextFormat(stream, tree, racks);
+    } else {
+      throw new BadFormatException("Bad format: " + format);
+    }
+  }
+
+  private void printJsonFormat(PrintStream stream, Map<String,
+      TreeSet<String>> tree, ArrayList<String> racks) throws IOException {
+    JsonFactory dumpFactory = new JsonFactory();
+    JsonGenerator dumpGenerator = dumpFactory.createGenerator(stream);
+    dumpGenerator.writeStartArray();
+
+    for(String r : racks) {
+      dumpGenerator.writeStartObject();
+      dumpGenerator.writeFieldName(r);
+      TreeSet<String> nodes = tree.get(r);
+      dumpGenerator.writeStartArray();
+
+      for(String n : nodes) {
+        dumpGenerator.writeStartObject();
+        dumpGenerator.writeStringField("ip", n);
+        String hostname = NetUtils.getHostNameOfIP(n);
+        if(hostname != null) {
+          dumpGenerator.writeStringField("hostname", hostname);
+        }
+        dumpGenerator.writeEndObject();
+      }
+      dumpGenerator.writeEndArray();
+      dumpGenerator.writeEndObject();
+    }
+    dumpGenerator.writeEndArray();
+    dumpGenerator.flush();
+
+    if (!dumpGenerator.isClosed()) {
+      dumpGenerator.close();
+    }
+  }
+
+  private void printTextFormat(PrintStream stream, Map<String,
+      TreeSet<String>> tree, ArrayList<String> racks) {
+    for(String r : racks) {
+      stream.println("Rack: " + r);
+      TreeSet<String> nodes = tree.get(r);
+
+      for(String n : nodes) {
+        stream.print("   " + n);
+        String hostname = NetUtils.getHostNameOfIP(n);
+        if(hostname != null) {
+          stream.print(" (" + hostname + ")");
+        }
+        stream.println();
+      }
+      stream.println();
+    }
+  }
+
+  @VisibleForTesting
+  static String parseAcceptHeader(HttpServletRequest request) {
+    String format = request.getHeader(HttpHeaders.ACCEPT);
+    return format != null && format.contains(FORMAT_JSON) ?
+            FORMAT_JSON : FORMAT_TEXT;
+  }
+
+  public static class BadFormatException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    public BadFormatException(String msg) {
+      super(msg);
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html

@@ -52,6 +52,7 @@
         <li><a href="jmx">Metrics</a></li>
         <li><a href="conf">Configuration</a></li>
         <li><a href="stacks">Process Thread Dump</a></li>
+        <li><a href="topology">Network Topology</a></li>
       </ul>
     </li>
   </ul>

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html

@@ -48,6 +48,7 @@
             <li><a href="jmx">Metrics</a></li>
             <li><a href="conf">Configuration</a></li>
             <li><a href="stacks">Process Thread Dump</a></li>
+            <li><a href="topology">Network Topology</a></li>
           </ul>
         </li>
       </ul>

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java

@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.StaticMapping;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestNetworkTopologyServlet {
+
+  @Test
+  public void testPrintTopologyTextFormat() throws IOException {
+    StaticMapping.resetMap();
+    Configuration conf = new HdfsConfiguration();
+    int dataNodesNum = 0;
+    final ArrayList<String> rackList = new ArrayList<String>();
+    for (int i = 0; i < 5; i++) {
+      for (int j = 0; j < 2; j++) {
+        rackList.add("/rack" + i);
+        dataNodesNum++;
+      }
+    }
+
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+            .numDataNodes(dataNodesNum)
+            .racks(rackList.toArray(new String[rackList.size()]))
+            .build();
+    cluster.waitActive();
+
+    // get http uri
+    String httpUri = cluster.getHttpUri(0);
+
+    // send http request
+    URL url = new URL(httpUri + "/topology");
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setReadTimeout(20000);
+    conn.setConnectTimeout(20000);
+    conn.connect();
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+    StringBuilder sb =
+        new StringBuilder("-- Network Topology -- \n");
+    sb.append(out);
+    sb.append("\n-- Network Topology -- ");
+    String topology = sb.toString();
+
+    // assert rack info
+    assertTrue(topology.contains("/rack0"));
+    assertTrue(topology.contains("/rack1"));
+    assertTrue(topology.contains("/rack2"));
+    assertTrue(topology.contains("/rack3"));
+    assertTrue(topology.contains("/rack4"));
+
+    // assert node number
+    assertEquals(topology.split("127.0.0.1").length - 1,
+        dataNodesNum);
+  }
+
+  @Test
+  public void testPrintTopologyJsonFormat() throws IOException {
+      StaticMapping.resetMap();
+      Configuration conf = new HdfsConfiguration();
+      int dataNodesNum = 0;
+      final ArrayList<String> rackList = new ArrayList<String>();
+      for (int i = 0; i < 5; i++) {
+          for (int j = 0; j < 2; j++) {
+              rackList.add("/rack" + i);
+              dataNodesNum++;
+          }
+      }
+
+      MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+              .numDataNodes(dataNodesNum)
+              .racks(rackList.toArray(new String[rackList.size()]))
+              .build();
+      cluster.waitActive();
+
+      // get http uri
+      String httpUri = cluster.getHttpUri(0);
+
+      // send http request
+      URL url = new URL(httpUri + "/topology");
+      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn.setReadTimeout(20000);
+      conn.setConnectTimeout(20000);
+      conn.setRequestProperty("Accept", "application/json");
+      conn.connect();
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+      String topology = out.toString();
+
+      // parse json
+      JsonNode racks = new ObjectMapper().readTree(topology);
+
+      // assert rack number
+      assertEquals(racks.size(), 5);
+
+      // assert node number
+      Iterator<JsonNode> elements = racks.elements();
+      int dataNodesCount = 0;
+      while(elements.hasNext()){
+          JsonNode rack = elements.next();
+          Iterator<Map.Entry<String, JsonNode>> fields = rack.fields();
+          while (fields.hasNext()) {
+              dataNodesCount += fields.next().getValue().size();
+          }
+      }
+      assertEquals(dataNodesCount, dataNodesNum);
+  }
+
+  @Test
+  public void testPrintTopologyNoDatanodesTextFormat() throws IOException {
+    StaticMapping.resetMap();
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+            .numDataNodes(0)
+            .build();
+    cluster.waitActive();
+
+    // get http uri
+    String httpUri = cluster.getHttpUri(0);
+
+    // send http request
+    URL url = new URL(httpUri + "/topology");
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setReadTimeout(20000);
+    conn.setConnectTimeout(20000);
+    conn.connect();
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+    StringBuilder sb =
+            new StringBuilder("-- Network Topology -- \n");
+    sb.append(out);
+    sb.append("\n-- Network Topology -- ");
+    String topology = sb.toString();
+
+    // assert node number
+    assertTrue(topology.contains("No DataNodes"));
+  }
+
+    @Test
+    public void testPrintTopologyNoDatanodesJsonFormat() throws IOException {
+        StaticMapping.resetMap();
+        Configuration conf = new HdfsConfiguration();
+        MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+                .numDataNodes(0)
+                .build();
+        cluster.waitActive();
+
+        // get http uri
+        String httpUri = cluster.getHttpUri(0);
+
+        // send http request
+        URL url = new URL(httpUri + "/topology");
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setReadTimeout(20000);
+        conn.setConnectTimeout(20000);
+        conn.setRequestProperty("Accept", "application/json");
+        conn.connect();
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+        StringBuilder sb =
+                new StringBuilder("-- Network Topology -- \n");
+        sb.append(out);
+        sb.append("\n-- Network Topology -- ");
+        String topology = sb.toString();
+
+        // assert node number
+        assertTrue(topology.contains("No DataNodes"));
+    }
+}