big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15358. RBF: Unify router datanode UI with namenode datanode UI. Contributed by Ayush Saxena.

Browse Source
This commit is contained in:
Ayush Saxena 2020-05-17 03:06:27 +05:30
parent 178336f8a8
commit 6e416a83d1
3 changed files with 179 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
View File

@ -23,6 +23,11 @@
<link rel="stylesheet" type="text/css" href="static/dataTables.bootstrap.css" /> <link rel="stylesheet" type="text/css" href="static/dataTables.bootstrap.css" />
<link rel="stylesheet" type="text/css" href="static/rbf.css" /> <link rel="stylesheet" type="text/css" href="static/rbf.css" />
<title>Router Information</title> <title>Router Information</title>
<style>
.col-sm-6 {
width: 30%;
}
</style>
</head> </head>
<body> <body>
@ -315,29 +320,44 @@
<ul class="dfshealth-node-legend"> <ul class="dfshealth-node-legend">
<li class="dfshealth-node-icon dfshealth-node-alive">In service</li> <li class="dfshealth-node-icon dfshealth-node-alive">In service</li>
<li class="dfshealth-node-icon dfshealth-node-down">Down</li> <li class="dfshealth-node-icon dfshealth-node-down">Down</li>
<li class="dfshealth-node-icon dfshealth-node-decommissioning">Decommissioning</li>
<li class="dfshealth-node-icon dfshealth-node-decommissioned">Decommissioned</li> <li class="dfshealth-node-icon dfshealth-node-decommissioned">Decommissioned</li>
<li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Decommissioned &amp; dead</li> <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Decommissioned &amp; dead</li>
</ul> </ul>
<ul class="dfshealth-node-legend">
<li class="dfshealth-node-icon dfshealth-node-entering-maintenance">Entering Maintenance</li>
<li class="dfshealth-node-icon dfshealth-node-in-maintenance">In Maintenance</li>
<li class="dfshealth-node-icon dfshealth-node-down-maintenance">In Maintenance &amp; dead</li>
</ul>
</div> </div>
<div class="page-header"><h1><small>Datanode usage histogram</small></h1></div>
<small><div id="datanode-usage-histogram"></div></small>
<div class="page-header"><h1><small>In operation</small></h1></div> <div class="page-header"><h1><small>In operation</small></h1></div>
<small> <small>
<p id="datanodefilter" class="col-sm-6">DataNode State </p>
<table class="table" id="table-datanodes"> <table class="table" id="table-datanodes">
<thead> <thead>
<tr> <tr>
<th>State</th>
<th>Node</th> <th>Node</th>
<th>Http Address</th>
<th>Last contact</th> <th>Last contact</th>
<th>Last Block Report</th>
<th>Used</th> <th>Used</th>
<th>Non DFS Used</th> <th>Non DFS Used</th>
<th style="width:180px; text-align:center">Capacity</th> <th style="width:180px; text-align:center">Capacity</th>
<!--th>Blocks</th--> <th>Blocks</th-->
<th>Block pool used</th> <th>Block pool used</th>
<!--th>Version</th--> <th>Version</th>
</tr> </tr>
</thead> </thead>
{#LiveNodes} {#LiveNodes}
<tr> <tr>
<td ng-value="{state}">{state}</td>
<td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td> <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
<td ng-value="{lastContact}">{#helper_relative_time value="{lastContact}"/}</td> <td ng-value="{state}-{name}"><a href='{dnWebAddress}'>{dnWebAddress}</a></td>
<td ng-value="{lastContact}">{lastContact}s</td>
<td ng-value="{lastBlockReport}">{lastBlockReport}m</td>
<td ng-value="{used}">{used|fmt_bytes}</td> <td ng-value="{used}">{used|fmt_bytes}</td>
<td ng-value="{nonDfsUsedSpace}">{nonDfsUsedSpace|fmt_bytes}</td> <td ng-value="{nonDfsUsedSpace}">{nonDfsUsedSpace|fmt_bytes}</td>
<td ng-value="{usedPercentage}" style="width:210px"> <td ng-value="{usedPercentage}" style="width:210px">
@ -349,26 +369,58 @@
</div> </div>
</div> </div>
</td> </td>
<!--td>{numBlocks}</td--> <td>{numBlocks}</td>
<td ng-value="{blockPoolUsedPercent}">{blockPoolUsed|fmt_bytes} ({blockPoolUsedPercent|fmt_percentage})</td> <td ng-value="{blockPoolUsedPercent}">{blockPoolUsed|fmt_bytes} ({blockPoolUsedPercent|fmt_percentage})</td>
<!--td>{version}</td--> <td>{version}</td>
</tr> </tr>
{/LiveNodes} {/LiveNodes}
{#DeadNodes} {#DeadNodes}
<tr class="danger"> <tr class="danger">
<td ng-value="{state}">{state}</td>
<td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td> <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
<td></td>
<td>{#helper_relative_time value="{lastContact}"/}</td> <td>{#helper_relative_time value="{lastContact}"/}</td>
<td></td> <td></td>
<!--td></td-->
<td></td> <td></td>
<!--td></td--> <td></td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr> </tr>
{/DeadNodes} {/DeadNodes}
</table> </table>
</small> </small>
<div class="page-header"><h1><small>Entering Maintenance</small></h1></div>
<small>
{?EnteringMaintenanceNodes}
<table class="table">
<thead>
<tr>
<th>Node</th>
<th>Under replicated blocks</th>
<th>Blocks with no live replicas</th>
<th>Under Replicated Blocks <br/>In files under construction</th>
</tr>
</thead>
{#EnteringMaintenanceNodes}
<tr>
<td>{name} ({xferaddr})</td>
<td>{underReplicatedBlocks}</td>
<td>{maintenanceOnlyReplicas}</td>
<td>{underReplicateInOpenFiles}</td>
</tr>
{/EnteringMaintenanceNodes}
</table>
{:else}
No nodes are entering maintenance.
{/EnteringMaintenanceNodes}
</small>
<div class="page-header"><h1><small>Decommissioning</small></h1></div> <div class="page-header"><h1><small>Decommissioning</small></h1></div>
<small> <small>
{?DecomNodes}
<table class="table"> <table class="table">
<thead> <thead>
<tr> <tr>
@ -387,6 +439,9 @@
</tr> </tr>
{/DecomNodes} {/DecomNodes}
</table> </table>
{:else}
No nodes are decommissioning.
{/DecomNodes}
</small> </small>
</script> </script>
@ -449,6 +504,7 @@
<script type="text/javascript" src="static/dust-full-2.0.0.min.js"></script> <script type="text/javascript" src="static/dust-full-2.0.0.min.js"></script>
<script type="text/javascript" src="static/dust-helpers-1.1.1.min.js"></script> <script type="text/javascript" src="static/dust-helpers-1.1.1.min.js"></script>
<script type="text/javascript" src="static/dfs-dust.js"></script> <script type="text/javascript" src="static/dfs-dust.js"></script>
<script type="text/javascript" src="/static/d3-v4.1.1.min.js"></script>
<script type="text/javascript" src="federationhealth.js"></script> <script type="text/javascript" src="federationhealth.js"></script>
</body> </body>
</html> </html>

122
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
View File

@ -278,20 +278,34 @@
for (var i = 0, e = nodes.length; i < e; ++i) { for (var i = 0, e = nodes.length; i < e; ++i) {
var n = nodes[i]; var n = nodes[i];
n.usedPercentage = Math.round((n.used + n.nonDfsUsedSpace) * 1.0 / n.capacity * 100); n.usedPercentage = Math.round((n.used + n.nonDfsUsedSpace) * 1.0 / n.capacity * 100);
var port = n.infoAddr.split(":")[1];
var securePort = n.infoSecureAddr.split(":")[1];
var dnHost = n.name.split(":")[0];
n.dnWebAddress = "http://" + dnHost + ":" + port;
if (securePort != 0) {
n.dnWebAddress = "https://" + dnHost + ":" + securePort;
}
if (n.adminState === "In Service") { if (n.adminState === "In Service") {
n.state = "alive"; n.state = "alive";
} else if (nodes[i].adminState === "Decommission In Progress") { } else if (nodes[i].adminState === "Decommission In Progress") {
n.state = "decommissioning"; n.state = "decommissioning";
} else if (nodes[i].adminState === "Decommissioned") { } else if (nodes[i].adminState === "Decommissioned") {
n.state = "decommissioned"; n.state = "decommissioned";
} else if (nodes[i].adminState === "Entering Maintenance") {
n.state = "entering-maintenance";
} else if (nodes[i].adminState === "In Maintenance") {
n.state = "in-maintenance";
} }
} }
} }
function augment_dead_nodes(nodes) { function augment_dead_nodes(nodes) {
for (var i = 0, e = nodes.length; i < e; ++i) { for (var i = 0, e = nodes.length; i < e; ++i) {
if (nodes[i].decommissioned) { if (nodes[i].adminState === "Decommissioned") {
nodes[i].state = "down-decommissioned"; nodes[i].state = "down-decommissioned";
} else if (nodes[i].adminState === "In Maintenance") {
nodes[i].state = "down-maintenance";
} else { } else {
nodes[i].state = "down"; nodes[i].state = "down";
} }
@ -303,9 +317,77 @@
r.DeadNodes = node_map_to_array(JSON.parse(r.DeadNodes)); r.DeadNodes = node_map_to_array(JSON.parse(r.DeadNodes));
augment_dead_nodes(r.DeadNodes); augment_dead_nodes(r.DeadNodes);
r.DecomNodes = node_map_to_array(JSON.parse(r.DecomNodes)); r.DecomNodes = node_map_to_array(JSON.parse(r.DecomNodes));
r.EnteringMaintenanceNodes = node_map_to_array(JSON.parse(r.EnteringMaintenanceNodes));
return r; return r;
} }
function renderHistogram(dnData) {
var data = dnData.LiveNodes.map(function(dn) {
return (dn.usedSpace / dn.capacity) * 100.0;
});
var formatCount = d3.format(",.0f");
var widthCap = $("div.container").width();
var heightCap = 150;
var margin = {top: 10, right: 60, bottom: 30, left: 30},
width = widthCap * 0.9,
height = heightCap - margin.top - margin.bottom;
var x = d3.scaleLinear()
.domain([0.0, 100.0])
.range([0, width]);
var bins = d3.histogram()
.domain(x.domain())
.thresholds(x.ticks(20))
(data);
var y = d3.scaleLinear()
.domain([0, d3.max(bins, function(d) { return d.length; })])
.range([height, 0]);
var svg = d3.select("#datanode-usage-histogram").append("svg")
.attr("width", width + 50.0)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
svg.append("text")
.attr("x", (width / 2))
.attr("y", heightCap - 6 - (margin.top / 2))
.attr("text-anchor", "middle")
.style("font-size", "15px")
.text("Disk usage of each DataNode (%)");
var bar = svg.selectAll(".bar")
.data(bins)
.enter().append("g")
.attr("class", "bar")
.attr("transform", function(d) { return "translate(" + x(d.x0) + "," + y(d.length) + ")"; });
window.liveNodes = dnData.LiveNodes;
bar.append("rect")
.attr("x", 1)
.attr("width", x(bins[0].x1) - x(bins[0].x0) - 1)
.attr("height", function(d) { return height - y(d.length); })
.attr("onclick", function (d) { return "open_hostip_list(" + d.x0 + "," + d.x1 + ")"; });
bar.append("text")
.attr("dy", ".75em")
.attr("y", 6)
.attr("x", (x(bins[0].x1) - x(bins[0].x0)) / 2)
.attr("text-anchor", "middle")
.text(function(d) { return formatCount(d.length); });
svg.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x));
}
$.get( $.get(
'jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo', 'jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo',
guard_with_startup_progress(function (resp) { guard_with_startup_progress(function (resp) {
@ -315,14 +397,38 @@
$('#tab-datanode').html(out); $('#tab-datanode').html(out);
$('#table-datanodes').dataTable( { $('#table-datanodes').dataTable( {
'lengthMenu': [ [25, 50, 100, -1], [25, 50, 100, "All"] ], 'lengthMenu': [ [25, 50, 100, -1], [25, 50, 100, "All"] ],
'columnDefs': [
{ 'targets': [ 0 ], 'visible': false, 'searchable': false }
],
'columns': [ 'columns': [
{ 'orderDataType': 'ng-value', 'searchable': true }, { 'orderDataType': 'ng-value', 'searchable': true , "defaultContent": "" },
{ 'orderDataType': 'ng-value', 'type': 'numeric' }, { 'orderDataType': 'ng-value', 'searchable': true , "defaultContent": "" },
{ 'orderDataType': 'ng-value', 'type': 'numeric' }, { 'orderDataType': 'ng-value', 'searchable': true , "defaultContent": ""},
{ 'orderDataType': 'ng-value', 'type': 'numeric' }, { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
{ 'orderDataType': 'ng-value', 'type': 'numeric' }, { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
{ 'orderDataType': 'ng-value', 'type': 'numeric'} { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
]}); { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
{ 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
{ 'type': 'num' , "defaultContent": 0},
{ 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0},
{ 'type': 'string' , "defaultContent": ""}
],
initComplete: function () {
var column = this.api().column([0]);
var select = $('<select class="datanodestatus form-control input-sm"><option value="">All</option></select>')
.appendTo('#datanodefilter')
.on('change', function () {
var val = $.fn.dataTable.util.escapeRegex(
$(this).val());
column.search(val ? '^' + val + '$' : '', true, false).draw();
});
console.log(select);
column.data().unique().sort().each(function (d, j) {
select.append('<option value="' + d + '">' + d + '</option>');
});
}
});
renderHistogram(data);
$('#ui-tabs a[href="#tab-datanode"]').tab('show'); $('#ui-tabs a[href="#tab-datanode"]').tab('show');
}); });
})).fail(ajax_error_handler); })).fail(ajax_error_handler);

2
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
View File

@ -371,6 +371,8 @@
<td></td> <td></td>
<td></td> <td></td>
<td></td> <td></td>
<td></td>
<td></td>>
</tr> </tr>
{/DeadNodes} {/DeadNodes}
</table> </table>