big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17331:Fix Blocks are always -1 and DataNode version are always UNKNOWN in federationhealth.html (#6429). Contributed by lei w.

Browse Source 
Signed-off-by: Shuyan Zhang <zhangshuyan@apache.org>
This commit is contained in:
Lei313 2024-01-18 21:10:54 +08:00 committed by GitHub
parent 4c3d4e6a57
commit cc4c4be1b7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 27 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
View File

@ -143,7 +143,7 @@ private DatanodeInfo(final String ipAddr, final String hostName,
final int xceiverCount, final String networkLocation,
final AdminStates adminState, final String upgradeDomain,
final long lastBlockReportTime, final long lastBlockReportMonotonic,
final int blockCount) {
final int blockCount, final String softwareVersion) {
super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
ipcPort);
this.capacity = capacity;
@ -162,6 +162,7 @@ private DatanodeInfo(final String ipAddr, final String hostName,
this.lastBlockReportTime = lastBlockReportTime;
this.lastBlockReportMonotonic = lastBlockReportMonotonic;
this.numBlocks = blockCount;
this.softwareVersion = softwareVersion;
}
/** Network location name. */
@ -699,6 +700,7 @@ public static class DatanodeInfoBuilder {
private long lastBlockReportTime = 0L;
private long lastBlockReportMonotonic = 0L;
private int numBlocks = 0;
private String softwareVersion;
// Please use setNumBlocks explicitly to set numBlocks as this method doesn't have
// sufficient info about numBlocks
@ -718,6 +720,9 @@ public DatanodeInfoBuilder setFrom(DatanodeInfo from) {
this.upgradeDomain = from.getUpgradeDomain();
this.lastBlockReportTime = from.getLastBlockReportTime();
this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic();
if (from.getSoftwareVersion() != null) {
this.softwareVersion = from.getSoftwareVersion();
}
setNodeID(from);
return this;
}
@ -844,18 +849,24 @@ public DatanodeInfoBuilder setLastBlockReportMonotonic(long time) {
this.lastBlockReportMonotonic = time;
return this;
}
public DatanodeInfoBuilder setNumBlocks(int blockCount) {
this.numBlocks = blockCount;
return this;
}
public DatanodeInfoBuilder setSoftwareVersion(String dnVersion) {
this.softwareVersion = dnVersion;
return this;
}
public DatanodeInfo build() {
return new DatanodeInfo(ipAddr, hostName, datanodeUuid, xferPort,
infoPort, infoSecurePort, ipcPort, capacity, dfsUsed, nonDfsUsed,
remaining, blockPoolUsed, cacheCapacity, cacheUsed, lastUpdate,
lastUpdateMonotonic, xceiverCount, location, adminState,
upgradeDomain, lastBlockReportTime, lastBlockReportMonotonic,
numBlocks);
numBlocks, softwareVersion);
}
}
}

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
View File

@ -385,6 +385,9 @@ public static DatanodeInfoProto convert(DatanodeInfo info) {
if (info.getUpgradeDomain() != null) {
builder.setUpgradeDomain(info.getUpgradeDomain());
}
if (info.getSoftwareVersion() != null) {
builder.setSoftwareVersion(info.getSoftwareVersion());
}
builder
.setId(convert((DatanodeID) info))
.setCapacity(info.getCapacity())
@ -786,7 +789,8 @@ static public DatanodeInfo convert(DatanodeInfoProto di) {
di.getLastBlockReportTime() : 0)
.setLastBlockReportMonotonic(di.hasLastBlockReportMonotonic() ?
di.getLastBlockReportMonotonic() : 0)
.setNumBlocks(di.getNumBlocks());
.setNumBlocks(di.getNumBlocks())
.setSoftwareVersion(di.getSoftwareVersion());
if (di.hasNonDfsUsed()) {
dinfo.setNonDfsUsed(di.getNonDfsUsed());

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
View File

@ -133,6 +133,7 @@ message DatanodeInfoProto {
optional uint64 lastBlockReportTime = 15 [default = 0];
optional uint64 lastBlockReportMonotonic = 16 [default = 0];
optional uint32 numBlocks = 17 [default = 0];
optional string softwareVersion = 18;
}
/**

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
View File

@ -481,7 +481,7 @@ private String getNodesImpl(final DatanodeReportType type) {
innerinfo.put("adminState", node.getAdminState().toString());
innerinfo.put("nonDfsUsedSpace", node.getNonDfsUsed());
innerinfo.put("capacity", node.getCapacity());
innerinfo.put("numBlocks", -1); // node.numBlocks()
innerinfo.put("numBlocks", node.getNumBlocks());
innerinfo.put("version", (node.getSoftwareVersion() == null ?
"UNKNOWN" : node.getSoftwareVersion()));
innerinfo.put("used", node.getDfsUsed());

9
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
View File

@ -1101,10 +1101,15 @@ private DatanodeStorageReport[] mergeDtanodeStorageReport(
DatanodeInfo dnInfo = dn.getDatanodeInfo();
String nodeId = dnInfo.getXferAddr();
DatanodeStorageReport oldDn = datanodesMap.get(nodeId);
if (oldDn == null ||
dnInfo.getLastUpdate() > oldDn.getDatanodeInfo().getLastUpdate()) {
if (oldDn == null) {
datanodesMap.put(nodeId, dn);
} else if (dnInfo.getLastUpdate() > oldDn.getDatanodeInfo().getLastUpdate()) {
dnInfo.setNumBlocks(dnInfo.getNumBlocks() +
oldDn.getDatanodeInfo().getNumBlocks());
datanodesMap.put(nodeId, dn);
} else {
oldDn.getDatanodeInfo().setNumBlocks(
oldDn.getDatanodeInfo().getNumBlocks() + dnInfo.getNumBlocks());
LOG.debug("{} is in multiple subclusters", nodeId);
}
}