big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-17260. Fix the logic for reconfigure slow peer enable for Namenode. (#6279). Contributed by huangzhaobo99.

Browse Source 
Reviewed-by: huhaiyang <huhaiyang926@126.com>
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
huangzhaobo 2023-12-03 17:39:55 +08:00 committed by GitHub
parent d0b460f270
commit 2323ad24a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 41 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
View File

@ -420,6 +420,7 @@ public class DatanodeManager {
LOG.error("Slow peers collection thread did not shutdown", e);
} finally {
slowPeerCollectorDaemon = null;
slowNodesUuidSet.clear();
}
}

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerDisabledTracker.java
View File

@ -33,7 +33,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.util.Timer;
/**
@ -46,11 +45,6 @@ public class SlowPeerDisabledTracker extends SlowPeerTracker {
public SlowPeerDisabledTracker(Configuration conf, Timer timer) {
super(conf, timer);
final boolean dataNodePeerStatsEnabledVal =
conf.getBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
Preconditions.checkArgument(!dataNodePeerStatsEnabledVal,
"SlowPeerDisabledTracker should only be used for disabled slow peer stats.");
}
@Override

40
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java
View File

@ -24,6 +24,8 @@ import org.apache.hadoop.hdfs.TestBlockStoragePolicy;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@ -32,7 +34,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Set;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@RunWith(Parameterized.class)
@ -136,4 +140,40 @@ public class TestReplicationPolicyExcludeSlowNodes
NameNode.LOG.info("Done working on it");
}
@Test
public void testSlowPeerTrackerEnabledClearSlowNodes() throws Exception {
namenode.getNamesystem().writeLock();
try {
// add nodes
for (DatanodeDescriptor dataNode : dataNodes) {
dnManager.addDatanode(dataNode);
}
// mock slow nodes
SlowPeerTracker tracker = dnManager.getSlowPeerTracker();
Assert.assertNotNull(tracker);
OutlierMetrics outlierMetrics = new OutlierMetrics(0.0, 0.0, 0.0, 5.0);
tracker.addReport(dataNodes[0].getInfoAddr(), dataNodes[3].getInfoAddr(),
outlierMetrics);
tracker.addReport(dataNodes[1].getInfoAddr(), dataNodes[3].getInfoAddr(),
outlierMetrics);
tracker.addReport(dataNodes[2].getInfoAddr(), dataNodes[3].getInfoAddr(),
outlierMetrics);
// check slow nodes
assertFalse(dnManager.isSlowPeerCollectorInitialized());
GenericTestUtils.waitFor(
() -> DatanodeManager.getSlowNodesUuidSet().size() == 3, 100, 3000);
// reconfig
namenode.reconfigureProperty(DFS_DATANODE_PEER_STATS_ENABLED_KEY,
"false");
assertTrue(dnManager.isSlowPeerCollectorInitialized());
assertEquals(0, DatanodeManager.getSlowNodesUuidSet().size());
} finally {
namenode.getNamesystem().writeUnlock();
}
}
}