HDFS-17019. Optimize the logic for reconfigure slow peer enable for Namenode" (#5671)
* HDFS-17019. Optimize the logic for reconfigure slow peer enable for Namenode
This commit is contained in:
parent
1dbaba8e70
commit
0c209961f8
@ -253,7 +253,6 @@ public class DatanodeManager {
|
|||||||
final boolean dataNodePeerStatsEnabledVal =
|
final boolean dataNodePeerStatsEnabledVal =
|
||||||
conf.getBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
|
conf.getBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
|
||||||
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
|
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
|
||||||
initSlowPeerTracker(conf, timer, dataNodePeerStatsEnabledVal);
|
|
||||||
this.maxSlowPeerReportNodes = conf.getInt(
|
this.maxSlowPeerReportNodes = conf.getInt(
|
||||||
DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY,
|
DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY,
|
||||||
DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT);
|
DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT);
|
||||||
@ -261,9 +260,7 @@ public class DatanodeManager {
|
|||||||
DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_KEY,
|
DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_KEY,
|
||||||
DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_DEFAULT,
|
DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_DEFAULT,
|
||||||
TimeUnit.MILLISECONDS);
|
TimeUnit.MILLISECONDS);
|
||||||
if (slowPeerTracker.isSlowPeerTrackerEnabled()) {
|
initSlowPeerTracker(conf, timer, dataNodePeerStatsEnabledVal);
|
||||||
startSlowPeerCollector();
|
|
||||||
}
|
|
||||||
this.slowDiskTracker = dataNodeDiskStatsEnabled ?
|
this.slowDiskTracker = dataNodeDiskStatsEnabled ?
|
||||||
new SlowDiskTracker(conf, timer) : null;
|
new SlowDiskTracker(conf, timer) : null;
|
||||||
this.defaultXferPort = NetUtils.createSocketAddr(
|
this.defaultXferPort = NetUtils.createSocketAddr(
|
||||||
@ -376,10 +373,16 @@ public void initSlowPeerTracker(Configuration conf, Timer timer,
|
|||||||
this.slowPeerTracker = dataNodePeerStatsEnabled ?
|
this.slowPeerTracker = dataNodePeerStatsEnabled ?
|
||||||
new SlowPeerTracker(conf, timer) :
|
new SlowPeerTracker(conf, timer) :
|
||||||
new SlowPeerDisabledTracker(conf, timer);
|
new SlowPeerDisabledTracker(conf, timer);
|
||||||
|
if (slowPeerTracker.isSlowPeerTrackerEnabled()) {
|
||||||
|
startSlowPeerCollector();
|
||||||
|
} else {
|
||||||
|
stopSlowPeerCollector();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void startSlowPeerCollector() {
|
private void startSlowPeerCollector() {
|
||||||
if (slowPeerCollectorDaemon != null) {
|
if (slowPeerCollectorDaemon != null) {
|
||||||
|
LOG.warn("Slow peers collection thread has been started.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
slowPeerCollectorDaemon = new Daemon(new Runnable() {
|
slowPeerCollectorDaemon = new Daemon(new Runnable() {
|
||||||
@ -402,9 +405,11 @@ public void run() {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
slowPeerCollectorDaemon.start();
|
slowPeerCollectorDaemon.start();
|
||||||
|
LOG.info("Slow peers collection thread start.");
|
||||||
}
|
}
|
||||||
|
|
||||||
public void stopSlowPeerCollector() {
|
public void stopSlowPeerCollector() {
|
||||||
|
LOG.info("Slow peers collection thread shutdown");
|
||||||
if (slowPeerCollectorDaemon == null) {
|
if (slowPeerCollectorDaemon == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -413,6 +418,8 @@ public void stopSlowPeerCollector() {
|
|||||||
slowPeerCollectorDaemon.join();
|
slowPeerCollectorDaemon.join();
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
LOG.error("Slow peers collection thread did not shutdown", e);
|
LOG.error("Slow peers collection thread did not shutdown", e);
|
||||||
|
} finally {
|
||||||
|
slowPeerCollectorDaemon = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2270,4 +2277,9 @@ public void setMaxSlowPeersToReport(int maxSlowPeersToReport) {
|
|||||||
Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned");
|
Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned");
|
||||||
slowPeerTracker.setMaxSlowPeersToReport(maxSlowPeersToReport);
|
slowPeerTracker.setMaxSlowPeersToReport(maxSlowPeersToReport);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
public boolean isSlowPeerCollectorInitialized() {
|
||||||
|
return slowPeerCollectorDaemon == null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -502,6 +502,7 @@ public void testSlowPeerTrackerEnabled() throws Exception {
|
|||||||
|
|
||||||
assertFalse("SlowNode tracker is already enabled. It should be disabled by default",
|
assertFalse("SlowNode tracker is already enabled. It should be disabled by default",
|
||||||
datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
|
datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
|
||||||
|
assertTrue(datanodeManager.isSlowPeerCollectorInitialized());
|
||||||
|
|
||||||
try {
|
try {
|
||||||
nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "non-boolean");
|
nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "non-boolean");
|
||||||
@ -515,6 +516,7 @@ public void testSlowPeerTrackerEnabled() throws Exception {
|
|||||||
nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "True");
|
nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "True");
|
||||||
assertTrue("SlowNode tracker is still disabled. Reconfiguration could not be successful",
|
assertTrue("SlowNode tracker is still disabled. Reconfiguration could not be successful",
|
||||||
datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
|
datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
|
||||||
|
assertFalse(datanodeManager.isSlowPeerCollectorInitialized());
|
||||||
|
|
||||||
nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, null);
|
nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, null);
|
||||||
assertFalse("SlowNode tracker is still enabled. Reconfiguration could not be successful",
|
assertFalse("SlowNode tracker is still enabled. Reconfiguration could not be successful",
|
||||||
|
Loading…
Reference in New Issue
Block a user