big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17634. RBF: Fix web UI missing DN last block report (#7080)

Browse Source
This commit is contained in:
Felix Nguyen 2024-10-22 19:54:16 +08:00 committed by GitHub
parent f931ede86b
commit 09b348753f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 25 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
View File

@ -25,6 +25,7 @@
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Function; import java.util.function.Function;
@ -476,6 +477,7 @@ private String getNodesImpl(final DatanodeReportType type) {
innerinfo.put("infoSecureAddr", node.getInfoSecureAddr()); innerinfo.put("infoSecureAddr", node.getInfoSecureAddr());
innerinfo.put("xferaddr", node.getXferAddr()); innerinfo.put("xferaddr", node.getXferAddr());
innerinfo.put("location", node.getNetworkLocation()); innerinfo.put("location", node.getNetworkLocation());
innerinfo.put("uuid", Optional.ofNullable(node.getDatanodeUuid()).orElse(""));
innerinfo.put("lastContact", getLastContact(node)); innerinfo.put("lastContact", getLastContact(node));
innerinfo.put("usedSpace", node.getDfsUsed()); innerinfo.put("usedSpace", node.getDfsUsed());
innerinfo.put("adminState", node.getAdminState().toString()); innerinfo.put("adminState", node.getAdminState().toString());
@ -492,6 +494,7 @@ private String getNodesImpl(final DatanodeReportType type) {
innerinfo.put("volfails", -1); // node.getVolumeFailures() innerinfo.put("volfails", -1); // node.getVolumeFailures()
innerinfo.put("blockPoolUsedPercentStdDev", innerinfo.put("blockPoolUsedPercentStdDev",
Util.getBlockPoolUsedPercentStdDev(storageReports)); Util.getBlockPoolUsedPercentStdDev(storageReports));
innerinfo.put("lastBlockReport", getLastBlockReport(node));
info.put(node.getXferAddrWithHostname(), info.put(node.getXferAddrWithHostname(),
Collections.unmodifiableMap(innerinfo)); Collections.unmodifiableMap(innerinfo));
} }
@ -795,6 +798,10 @@ private long getLastContact(DatanodeInfo node) {
return (now() - node.getLastUpdate()) / 1000; return (now() - node.getLastUpdate()) / 1000;
} }
private long getLastBlockReport(DatanodeInfo node) {
return (now() - node.getLastBlockReportTime()) / 60000;
}
///////////////////////////////////////////////////////// /////////////////////////////////////////////////////////
// NameNodeStatusMXBean // NameNodeStatusMXBean
///////////////////////////////////////////////////////// /////////////////////////////////////////////////////////

18
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
View File

@ -135,6 +135,8 @@
import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.test.LambdaTestUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import org.junit.After; import org.junit.After;
@ -1880,6 +1882,22 @@ public void testNamenodeMetrics() throws Exception {
JSONObject jsonObject = new JSONObject(jsonString0); JSONObject jsonObject = new JSONObject(jsonString0);
assertEquals(NUM_SUBCLUSTERS * NUM_DNS, jsonObject.names().length()); assertEquals(NUM_SUBCLUSTERS * NUM_DNS, jsonObject.names().length());
JSONObject jsonObjectNn =
new JSONObject(cluster.getRandomNamenode().getNamenode().getNamesystem().getLiveNodes());
// DN report by NN and router should be the same
String randomDn = (String) jsonObjectNn.names().get(0);
JSONObject randomReportNn = jsonObjectNn.getJSONObject(randomDn);
JSONObject randomReportRouter = jsonObject.getJSONObject(randomDn);
JSONArray keys = randomReportNn.names();
for (int i = 0; i < keys.length(); i++) {
String key = keys.getString(i);
// Skip the 2 keys that always return -1
if (key.equals("blockScheduled") || key.equals("volfails")) {
continue;
}
assertEquals(randomReportRouter.get(key), randomReportNn.get(key));
}
// We should be caching this information // We should be caching this information
String jsonString1 = metrics.getLiveNodes(); String jsonString1 = metrics.getLiveNodes();
assertEquals(jsonString0, jsonString1); assertEquals(jsonString0, jsonString1);