big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17019. Optimize the logic for reconfigure slow peer enable for Namenode" (#5671)

Browse Source 
* HDFS-17019. Optimize the logic for reconfigure slow peer enable for Namenode
This commit is contained in:
huhaiyang 2023-06-08 10:05:49 +08:00 committed by GitHub
parent 1dbaba8e70
commit 0c209961f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
View File

@ -253,7 +253,6 @@ public class DatanodeManager {
final boolean dataNodePeerStatsEnabledVal =
conf.getBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
initSlowPeerTracker(conf, timer, dataNodePeerStatsEnabledVal);
this.maxSlowPeerReportNodes = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT);
@ -261,9 +260,7 @@ public class DatanodeManager {
DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_KEY,
DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
if (slowPeerTracker.isSlowPeerTrackerEnabled()) {
startSlowPeerCollector();
}
initSlowPeerTracker(conf, timer, dataNodePeerStatsEnabledVal);
this.slowDiskTracker = dataNodeDiskStatsEnabled ?
new SlowDiskTracker(conf, timer) : null;
this.defaultXferPort = NetUtils.createSocketAddr(
@ -376,10 +373,16 @@ public void initSlowPeerTracker(Configuration conf, Timer timer,
this.slowPeerTracker = dataNodePeerStatsEnabled ?
new SlowPeerTracker(conf, timer) :
new SlowPeerDisabledTracker(conf, timer);
if (slowPeerTracker.isSlowPeerTrackerEnabled()) {
startSlowPeerCollector();
} else {
stopSlowPeerCollector();
}
}
private void startSlowPeerCollector() {
if (slowPeerCollectorDaemon != null) {
LOG.warn("Slow peers collection thread has been started.");
return;
}
slowPeerCollectorDaemon = new Daemon(new Runnable() {
@ -402,9 +405,11 @@ public void run() {
}
});
slowPeerCollectorDaemon.start();
LOG.info("Slow peers collection thread start.");
}
public void stopSlowPeerCollector() {
LOG.info("Slow peers collection thread shutdown");
if (slowPeerCollectorDaemon == null) {
return;
}
@ -413,6 +418,8 @@ public void stopSlowPeerCollector() {
slowPeerCollectorDaemon.join();
} catch (InterruptedException e) {
LOG.error("Slow peers collection thread did not shutdown", e);
} finally {
slowPeerCollectorDaemon = null;
}
}
@ -2270,4 +2277,9 @@ public void setMaxSlowPeersToReport(int maxSlowPeersToReport) {
Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned");
slowPeerTracker.setMaxSlowPeersToReport(maxSlowPeersToReport);
}
@VisibleForTesting
public boolean isSlowPeerCollectorInitialized() {
return slowPeerCollectorDaemon == null;
}
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
View File

@ -502,6 +502,7 @@ public void testSlowPeerTrackerEnabled() throws Exception {
assertFalse("SlowNode tracker is already enabled. It should be disabled by default",
datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
assertTrue(datanodeManager.isSlowPeerCollectorInitialized());
try {
nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "non-boolean");
@ -515,6 +516,7 @@ public void testSlowPeerTrackerEnabled() throws Exception {
nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "True");
assertTrue("SlowNode tracker is still disabled. Reconfiguration could not be successful",
datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
assertFalse(datanodeManager.isSlowPeerCollectorInitialized());
nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, null);
assertFalse("SlowNode tracker is still enabled. Reconfiguration could not be successful",