big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16269. [Fix] Improve NNThroughputBenchmark#blockReport operation. (#3544)

Browse Source 
Reviewed-by: Fei Hui <feihui.ustc@gmail.com>
Reviewed-by: Wei-Chiu Chuang <weichiu@apache.org>
Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
(cherry picked from commit 618fea27d2)

 Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
This commit is contained in:
jianghuazhu 2021-11-01 23:55:50 +08:00 committed by Akira Ajisaka
parent 7e57fc053a
commit 5fbd9d333e
2 changed files with 40 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View File

@ -1173,10 +1173,28 @@ void generateInputs(int[] ignore) throws IOException {
private ExtendedBlock addBlocks(String fileName, String clientName)
throws IOException {
DatanodeInfo[] excludeNodes = null;
DatanodeInfo[] dnInfos = clientProto.getDatanodeReport(
HdfsConstants.DatanodeReportType.LIVE);
if (dnInfos != null && dnInfos.length > 0) {
List<DatanodeInfo> tmpNodes = new ArrayList<>();
String localHost = DNS.getDefaultHost("default", "default");
for (DatanodeInfo dnInfo : dnInfos) {
if (!localHost.equals(dnInfo.getHostName()) ||
(dnInfo.getXferPort() > datanodes.length)) {
tmpNodes.add(dnInfo);
}
}
if (tmpNodes.size() > 0) {
excludeNodes = tmpNodes.toArray(new DatanodeInfo[tmpNodes.size()]);
}
}
ExtendedBlock prevBlock = null;
for(int jdx = 0; jdx < blocksPerFile; jdx++) {
LocatedBlock loc = addBlock(fileName, clientName,
prevBlock, null, HdfsConstants.GRANDFATHER_INODE_ID, null);
prevBlock, excludeNodes, HdfsConstants.GRANDFATHER_INODE_ID, null);
prevBlock = loc.getBlock();
for(DatanodeInfo dnInfo : loc.getLocations()) {
int dnIdx = dnInfo.getXferPort() - 1;

21
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
View File

@ -120,4 +120,25 @@ public void testNNThroughputRemoteAgainstNNWithFsOption() throws Exception {
}
}
}
/**
* This test runs {@link NNThroughputBenchmark} against a mini DFS cluster
* for block report operation.
*/
@Test(timeout = 120000)
public void testNNThroughputForBlockReportOp() throws Exception {
final Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).
numDataNodes(3).build()) {
cluster.waitActive();
final Configuration benchConf = new HdfsConfiguration();
benchConf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
benchConf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
NNThroughputBenchmark.runBenchmark(benchConf,
new String[]{"-fs", cluster.getURI().toString(), "-op",
"blockReport", "-datanodes", "3", "-reports", "2"});
}
}
}