big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16314. Support to make dfs.namenode.block-placement-policy.exclude-slow-nodes.enabled reconfigurable (#3664)

Browse Source 
Reviewed-by: Fei Hui <feihui.ustc@gmail.com>
Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
(cherry picked from commit c0f405a46b)
This commit is contained in:
huhaiyang 2021-12-04 01:19:36 +08:00 committed by Akira Ajisaka
parent bcbe4fbfa7
commit a24c98163f
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
6 changed files with 77 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -5319,4 +5319,14 @@ public void disableSPS() {
public StoragePolicySatisfyManager getSPSManager() {
return spsManager;
}
public void setExcludeSlowNodesEnabled(boolean enable) {
placementPolicies.getPolicy(CONTIGUOUS).setExcludeSlowNodesEnabled(enable);
placementPolicies.getPolicy(STRIPED).setExcludeSlowNodesEnabled(enable);
}
@VisibleForTesting
public boolean getExcludeSlowNodesEnabled(BlockType blockType) {
return placementPolicies.getPolicy(blockType).getExcludeSlowNodesEnabled();
}
}

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
View File

@ -261,4 +261,16 @@ public <T> void splitNodesWithRack(
}
}
}
/**
* Updates the value used for excludeSlowNodesEnabled, which is set by
* {@code DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY}
* initially.
*
* @param enable true, we will filter out slow nodes
* when choosing targets for blocks, otherwise false not filter.
*/
public abstract void setExcludeSlowNodesEnabled(boolean enable);
public abstract boolean getExcludeSlowNodesEnabled();
}

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
View File

@ -103,7 +103,7 @@ private String getText() {
protected double considerLoadFactor;
private boolean preferLocalNode;
private boolean dataNodePeerStatsEnabled;
private boolean excludeSlowNodesEnabled;
private volatile boolean excludeSlowNodesEnabled;
protected NetworkTopology clusterMap;
protected Host2NodesMap host2datanodeMap;
private FSClusterStats stats;
@ -1359,5 +1359,15 @@ protected Collection<DatanodeStorageInfo> pickupReplicaSet(
void setPreferLocalNode(boolean prefer) {
this.preferLocalNode = prefer;
}
@Override
public void setExcludeSlowNodesEnabled(boolean enable) {
this.excludeSlowNodesEnabled = enable;
}
@Override
public boolean getExcludeSlowNodesEnabled() {
return excludeSlowNodesEnabled;
}
}

27
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
View File

@ -188,6 +188,8 @@
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT;
import static org.apache.hadoop.util.ExitUtil.terminate;
import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
@ -330,7 +332,8 @@ public enum OperationCategory {
DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY,
DFS_IMAGE_PARALLEL_LOAD_KEY,
DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY));
DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY,
DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY));
private static final String USAGE = "Usage: hdfs namenode ["
+ StartupOption.BACKUP.getName() + "] | \n\t["
@ -2196,7 +2199,8 @@ protected String reconfigurePropertyImpl(String property, String newVal)
return newVal;
} else if (property.equals(DFS_IMAGE_PARALLEL_LOAD_KEY)) {
return reconfigureParallelLoad(newVal);
} else if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)) {
} else if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)
|| (property.equals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY))) {
return reconfigureSlowNodesParameters(datanodeManager, property, newVal);
} else {
throw new ReconfigurationException(property, newVal, getConf().get(
@ -2230,7 +2234,7 @@ private String reconfReplicationParameters(final String newVal,
newSetting = bm.getBlocksReplWorkMultiplier();
} else {
throw new IllegalArgumentException("Unexpected property " +
property + "in reconfReplicationParameters");
property + " in reconfReplicationParameters");
}
LOG.info("RECONFIGURE* changed {} to {}", property, newSetting);
return String.valueOf(newSetting);
@ -2386,15 +2390,24 @@ String reconfigureParallelLoad(String newVal) {
String reconfigureSlowNodesParameters(final DatanodeManager datanodeManager,
final String property, final String newVal) throws ReconfigurationException {
BlockManager bm = namesystem.getBlockManager();
namesystem.writeLock();
boolean enable;
try {
if (newVal == null) {
enable = DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT;
if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)) {
enable = (newVal == null ? DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT :
Boolean.parseBoolean(newVal));
datanodeManager.setAvoidSlowDataNodesForReadEnabled(enable);
} else if (property.equals(
DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY)) {
enable = (newVal == null ?
DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT :
Boolean.parseBoolean(newVal));
bm.setExcludeSlowNodesEnabled(enable);
} else {
enable = Boolean.parseBoolean(newVal);
throw new IllegalArgumentException("Unexpected property " +
property + " in reconfigureSlowNodesParameters");
}
datanodeManager.setAvoidSlowDataNodesForReadEnabled(enable);
LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
return Boolean.toString(enable);
} catch (IllegalArgumentException e) {

22
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
View File

@ -34,10 +34,12 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode;
import org.apache.hadoop.hdfs.protocol.BlockType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfyManager;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.test.GenericTestUtils;
@ -52,6 +54,7 @@
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT;
public class TestNameNodeReconfigure {
@ -399,8 +402,8 @@ public void testEnableParallelLoadAfterReconfigured()
public void testEnableSlowNodesParametersAfterReconfigured()
throws ReconfigurationException {
final NameNode nameNode = cluster.getNameNode();
final DatanodeManager datanodeManager = nameNode.namesystem
.getBlockManager().getDatanodeManager();
final BlockManager blockManager = nameNode.namesystem.getBlockManager();
final DatanodeManager datanodeManager = blockManager.getDatanodeManager();
// By default, avoidSlowDataNodesForRead is false.
assertEquals(false, datanodeManager.getEnableAvoidSlowDataNodesForRead());
@ -410,6 +413,21 @@ public void testEnableSlowNodesParametersAfterReconfigured()
// After reconfigured, avoidSlowDataNodesForRead is true.
assertEquals(true, datanodeManager.getEnableAvoidSlowDataNodesForRead());
// By default, excludeSlowNodesEnabled is false.
assertEquals(false, blockManager.
getExcludeSlowNodesEnabled(BlockType.CONTIGUOUS));
assertEquals(false, blockManager.
getExcludeSlowNodesEnabled(BlockType.STRIPED));
nameNode.reconfigureProperty(
DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, Boolean.toString(true));
// After reconfigured, excludeSlowNodesEnabled is true.
assertEquals(true, blockManager.
getExcludeSlowNodesEnabled(BlockType.CONTIGUOUS));
assertEquals(true, blockManager.
getExcludeSlowNodesEnabled(BlockType.STRIPED));
}
@After

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
View File

@ -26,6 +26,7 @@
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
import java.util.function.Supplier;
import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
@ -423,13 +424,14 @@ public void testNameNodeGetReconfigurableProperties() throws IOException {
final List<String> outs = Lists.newArrayList();
final List<String> errs = Lists.newArrayList();
getReconfigurableProperties("namenode", address, outs, errs);
assertEquals(14, outs.size());
assertEquals(15, outs.size());
assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(1));
assertEquals(DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, outs.get(2));
assertEquals(DFS_HEARTBEAT_INTERVAL_KEY, outs.get(3));
assertEquals(DFS_IMAGE_PARALLEL_LOAD_KEY, outs.get(4));
assertEquals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, outs.get(5));
assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(6));
assertEquals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, outs.get(6));
assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(7));
assertEquals(errs.size(), 0);
}