HDFS-15358. RBF: Unify router datanode UI with namenode datanode UI. Contributed by Ayush Saxena.

2020-05-17 03:06:27 +05:30 · 2020-05-17 03:06:27 +05:30 · 6e416a83d1
commit 6e416a83d1
parent 178336f8a8
3 changed files with 179 additions and 15 deletions
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
@ -23,6 +23,11 @@
 <link rel="stylesheet" type="text/css" href="static/dataTables.bootstrap.css" />
 <link rel="stylesheet" type="text/css" href="static/rbf.css" />
 <title>Router Information</title>
+<style>
+  .col-sm-6 {
+    width: 30%;
+  }
+</style>
 </head>
 <body>

@ -315,29 +320,44 @@
  <ul class="dfshealth-node-legend">
    <li class="dfshealth-node-icon dfshealth-node-alive">In service</li>
    <li class="dfshealth-node-icon dfshealth-node-down">Down</li>
+    <li class="dfshealth-node-icon dfshealth-node-decommissioning">Decommissioning</li>
    <li class="dfshealth-node-icon dfshealth-node-decommissioned">Decommissioned</li>
    <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Decommissioned &amp; dead</li>
  </ul>
+  <ul class="dfshealth-node-legend">
+    <li class="dfshealth-node-icon dfshealth-node-entering-maintenance">Entering Maintenance</li>
+    <li class="dfshealth-node-icon dfshealth-node-in-maintenance">In Maintenance</li>
+    <li class="dfshealth-node-icon dfshealth-node-down-maintenance">In Maintenance &amp; dead</li>
+  </ul>
 </div>
+<div class="page-header"><h1><small>Datanode usage histogram</small></h1></div>
+<small><div id="datanode-usage-histogram"></div></small>
 <div class="page-header"><h1><small>In operation</small></h1></div>
 <small>
+<p id="datanodefilter" class="col-sm-6">DataNode State </p>
 <table class="table" id="table-datanodes">
  <thead>
    <tr>
+      <th>State</th>
      <th>Node</th>
+      <th>Http Address</th>
      <th>Last contact</th>
+      <th>Last Block Report</th>
      <th>Used</th>
      <th>Non DFS Used</th>
      <th style="width:180px; text-align:center">Capacity</th>
-      <!--th>Blocks</th-->
+      <th>Blocks</th-->
      <th>Block pool used</th>
-      <!--th>Version</th-->
+      <th>Version</th>
    </tr>
  </thead>
  {#LiveNodes}
  <tr>
+    <td ng-value="{state}">{state}</td>
    <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
-    <td ng-value="{lastContact}">{#helper_relative_time value="{lastContact}"/}</td>
+    <td ng-value="{state}-{name}"><a href='{dnWebAddress}'>{dnWebAddress}</a></td>
+    <td ng-value="{lastContact}">{lastContact}s</td>
+    <td ng-value="{lastBlockReport}">{lastBlockReport}m</td>
    <td ng-value="{used}">{used|fmt_bytes}</td>
    <td ng-value="{nonDfsUsedSpace}">{nonDfsUsedSpace|fmt_bytes}</td>
    <td ng-value="{usedPercentage}" style="width:210px">
@ -349,26 +369,58 @@
        </div>
      </div>
    </td>
-    <!--td>{numBlocks}</td-->
+    <td>{numBlocks}</td>
    <td ng-value="{blockPoolUsedPercent}">{blockPoolUsed|fmt_bytes} ({blockPoolUsedPercent|fmt_percentage})</td>
-    <!--td>{version}</td-->
+    <td>{version}</td>
  </tr>
  {/LiveNodes}
  {#DeadNodes}
  <tr class="danger">
+    <td ng-value="{state}">{state}</td>
    <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
+    <td></td>
    <td>{#helper_relative_time value="{lastContact}"/}</td>
    <td></td>
-    <!--td></td-->
    <td></td>
-    <!--td></td-->
+    <td></td>
+    <td></td>
+    <td></td>
+    <td></td>
+    <td></td>
  </tr>
  {/DeadNodes}
 </table>
 </small>

+<div class="page-header"><h1><small>Entering Maintenance</small></h1></div>
+<small>
+  {?EnteringMaintenanceNodes}
+  <table class="table">
+    <thead>
+    <tr>
+      <th>Node</th>
+      <th>Under replicated blocks</th>
+      <th>Blocks with no live replicas</th>
+      <th>Under Replicated Blocks <br/>In files under construction</th>
+    </tr>
+    </thead>
+    {#EnteringMaintenanceNodes}
+    <tr>
+      <td>{name} ({xferaddr})</td>
+      <td>{underReplicatedBlocks}</td>
+      <td>{maintenanceOnlyReplicas}</td>
+      <td>{underReplicateInOpenFiles}</td>
+    </tr>
+    {/EnteringMaintenanceNodes}
+  </table>
+  {:else}
+  No nodes are entering maintenance.
+  {/EnteringMaintenanceNodes}
+</small>
+
 <div class="page-header"><h1><small>Decommissioning</small></h1></div>
 <small>
+{?DecomNodes}
 <table class="table">
  <thead>
    <tr>
@ -387,6 +439,9 @@
  </tr>
  {/DecomNodes}
 </table>
+{:else}
+No nodes are decommissioning.
+{/DecomNodes}
 </small>
 </script>

@ -449,6 +504,7 @@
 <script type="text/javascript" src="static/dust-full-2.0.0.min.js"></script>
 <script type="text/javascript" src="static/dust-helpers-1.1.1.min.js"></script>
 <script type="text/javascript" src="static/dfs-dust.js"></script>
+<script type="text/javascript" src="/static/d3-v4.1.1.min.js"></script>
 <script type="text/javascript" src="federationhealth.js"></script>
 </body>
 </html>
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
@ -278,20 +278,34 @@
        for (var i = 0, e = nodes.length; i < e; ++i) {
          var n = nodes[i];
          n.usedPercentage = Math.round((n.used + n.nonDfsUsedSpace) * 1.0 / n.capacity * 100);
+          var port = n.infoAddr.split(":")[1];
+          var securePort = n.infoSecureAddr.split(":")[1];
+          var dnHost = n.name.split(":")[0];
+          n.dnWebAddress = "http://" + dnHost + ":" + port;
+          if (securePort != 0) {
+            n.dnWebAddress = "https://" + dnHost + ":" + securePort;
+          }
+
          if (n.adminState === "In Service") {
            n.state = "alive";
          } else if (nodes[i].adminState === "Decommission In Progress") {
            n.state = "decommissioning";
          } else if (nodes[i].adminState === "Decommissioned") {
            n.state = "decommissioned";
+          } else if (nodes[i].adminState === "Entering Maintenance") {
+            n.state = "entering-maintenance";
+          } else if (nodes[i].adminState === "In Maintenance") {
+            n.state = "in-maintenance";
          }
        }
      }

      function augment_dead_nodes(nodes) {
        for (var i = 0, e = nodes.length; i < e; ++i) {
-          if (nodes[i].decommissioned) {
+          if (nodes[i].adminState === "Decommissioned") {
            nodes[i].state = "down-decommissioned";
+          } else if (nodes[i].adminState === "In Maintenance") {
+            nodes[i].state = "down-maintenance";
          } else {
            nodes[i].state = "down";
          }
@ -303,9 +317,77 @@
      r.DeadNodes = node_map_to_array(JSON.parse(r.DeadNodes));
      augment_dead_nodes(r.DeadNodes);
      r.DecomNodes = node_map_to_array(JSON.parse(r.DecomNodes));
+      r.EnteringMaintenanceNodes = node_map_to_array(JSON.parse(r.EnteringMaintenanceNodes));
      return r;
    }

+    function renderHistogram(dnData) {
+      var data = dnData.LiveNodes.map(function(dn) {
+        return (dn.usedSpace / dn.capacity) * 100.0;
+      });
+
+      var formatCount = d3.format(",.0f");
+
+      var widthCap = $("div.container").width();
+      var heightCap = 150;
+
+      var margin = {top: 10, right: 60, bottom: 30, left: 30},
+          width = widthCap * 0.9,
+          height = heightCap - margin.top - margin.bottom;
+
+      var x = d3.scaleLinear()
+          .domain([0.0, 100.0])
+          .range([0, width]);
+
+      var bins = d3.histogram()
+          .domain(x.domain())
+          .thresholds(x.ticks(20))
+          (data);
+
+      var y = d3.scaleLinear()
+          .domain([0, d3.max(bins, function(d) { return d.length; })])
+          .range([height, 0]);
+
+      var svg = d3.select("#datanode-usage-histogram").append("svg")
+          .attr("width", width + 50.0)
+          .attr("height", height + margin.top + margin.bottom)
+          .append("g")
+          .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+      svg.append("text")
+          .attr("x", (width / 2))
+          .attr("y", heightCap - 6 - (margin.top / 2))
+          .attr("text-anchor", "middle")
+          .style("font-size", "15px")
+          .text("Disk usage of each DataNode (%)");
+
+      var bar = svg.selectAll(".bar")
+          .data(bins)
+          .enter().append("g")
+          .attr("class", "bar")
+          .attr("transform", function(d) { return "translate(" + x(d.x0) + "," + y(d.length) + ")"; });
+
+      window.liveNodes = dnData.LiveNodes;
+
+      bar.append("rect")
+          .attr("x", 1)
+          .attr("width", x(bins[0].x1) - x(bins[0].x0) - 1)
+          .attr("height", function(d) { return height - y(d.length); })
+          .attr("onclick", function (d) { return "open_hostip_list(" + d.x0 + "," + d.x1 + ")"; });
+
+      bar.append("text")
+          .attr("dy", ".75em")
+          .attr("y", 6)
+          .attr("x", (x(bins[0].x1) - x(bins[0].x0)) / 2)
+          .attr("text-anchor", "middle")
+          .text(function(d) { return formatCount(d.length); });
+
+      svg.append("g")
+          .attr("class", "axis axis--x")
+          .attr("transform", "translate(0," + height + ")")
+          .call(d3.axisBottom(x));
+    }
+
    $.get(
      'jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo',
      guard_with_startup_progress(function (resp) {
@ -315,14 +397,38 @@
          $('#tab-datanode').html(out);
          $('#table-datanodes').dataTable( {
            'lengthMenu': [ [25, 50, 100, -1], [25, 50, 100, "All"] ],
+            'columnDefs': [
+              { 'targets': [ 0 ], 'visible': false, 'searchable': false }
+            ],
            'columns': [
-              { 'orderDataType': 'ng-value', 'searchable': true },
-              { 'orderDataType': 'ng-value', 'type': 'numeric' },
-              { 'orderDataType': 'ng-value', 'type': 'numeric' },
-              { 'orderDataType': 'ng-value', 'type': 'numeric' },
-              { 'orderDataType': 'ng-value', 'type': 'numeric' },
-              { 'orderDataType': 'ng-value', 'type': 'numeric'}
-            ]});
+              { 'orderDataType': 'ng-value', 'searchable': true , "defaultContent": "" },
+              { 'orderDataType': 'ng-value', 'searchable': true , "defaultContent": "" },
+              { 'orderDataType': 'ng-value', 'searchable': true , "defaultContent": ""},
+              { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
+              { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
+              { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
+              { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
+              { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
+              { 'type': 'num' , "defaultContent": 0},
+              { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
+              { 'type': 'string' , "defaultContent": ""}
+            ],
+            initComplete: function () {
+              var column = this.api().column([0]);
+              var select = $('<select class="datanodestatus form-control input-sm"><option value="">All</option></select>')
+                  .appendTo('#datanodefilter')
+                  .on('change', function () {
+                    var val = $.fn.dataTable.util.escapeRegex(
+                        $(this).val());
+                    column.search(val ? '^' + val + '$' : '', true, false).draw();
+                  });
+              console.log(select);
+              column.data().unique().sort().each(function (d, j) {
+                select.append('<option value="' + d + '">' + d + '</option>');
+              });
+            }
+          });
+          renderHistogram(data);
          $('#ui-tabs a[href="#tab-datanode"]').tab('show');
        });
      })).fail(ajax_error_handler);
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@ -371,6 +371,8 @@
    <td></td>
    <td></td>
    <td></td>
+    <td></td>
+    <td></td>>
  </tr>
  {/DeadNodes}
 </table>