big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17137. Standby/Observer NameNode skip to handle redundant replica block logic when set decrease replication. (#5913). Contributed by Haiyang Hu.

Browse Source 
Reviewed-by: Tao Li <tomscut@apache.org>
Signed-off-by: He Xiaoqiao <hexiaoqiao@apache.org>
This commit is contained in:
huhaiyang 2023-08-08 15:42:23 +08:00 committed by GitHub
parent b5b9a81538
commit 5b81caf0cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 59 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -4030,6 +4030,11 @@ public class BlockManager implements BlockStatsMXBean {
// update neededReconstruction priority queues
b.setReplication(newRepl);
// Process the block only when active NN is out of safe mode.
if (!isPopulatingReplQueues()) {
return;
}
NumberReplicas num = countNodes(b);
updateNeededReconstructions(b, 0, newRepl - oldRepl);
if (shouldProcessExtraRedundancy(num, newRepl)) {

54
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java
View File

@ -97,4 +97,58 @@ public class TestStandbyBlockManagement {
}
}
/**
* Test Standby/Observer NameNode should not handle redundant replica block logic
* when set decrease replication.
* @throws Exception
*/
@Test(timeout = 60000)
public void testNotHandleRedundantReplica() throws Exception {
Configuration conf = new Configuration();
HAUtil.setAllowStandbyReads(conf, true);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
// Create HA Cluster.
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(4).build()) {
cluster.waitActive();
cluster.transitionToActive(0);
NameNode nn1 = cluster.getNameNode(0);
assertEquals("ACTIVE", nn1.getNamesystem().getState().name());
NameNode nn2 = cluster.getNameNode(1);
assertEquals("STANDBY", nn2.getNamesystem().getState().name());
cluster.triggerHeartbeats();
// Sending the FBR.
cluster.triggerBlockReports();
// Default excessRedundancyMap size as 0.
assertEquals(0, nn1.getNamesystem().getBlockManager().getExcessBlocksCount());
assertEquals(0, nn2.getNamesystem().getBlockManager().getExcessBlocksCount());
FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf);
// Create test file.
Path file = new Path("/test");
long fileLength = 512;
DFSTestUtil.createFile(fs, file, fileLength, (short) 4, 0L);
DFSTestUtil.waitReplication(fs, file, (short) 4);
// Set decrease 3 replication.
fs.setReplication(file, (short) 3);
HATestUtil.waitForStandbyToCatchUp(nn1, nn2);
// Make sure the DN has deleted the block and report to NNs.
cluster.triggerHeartbeats();
HATestUtil.waitForDNDeletions(cluster);
cluster.triggerDeletionReports();
DFSTestUtil.waitReplication(fs, file, (short) 3);
// Delete excess replica, active and standby nn excessRedundancyMap size as 0.
assertEquals(0, nn1.getNamesystem().getBlockManager().getExcessBlocksCount());
assertEquals(0, nn2.getNamesystem().getBlockManager().getExcessBlocksCount());
}
}
}