big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17574. Make NNThroughputBenchmark support human-friendly units about blocksize. (#6931). Contributed by wangzhongwei.

Browse Source 
Signed-off-by: He Xiaoqiao <hexiaoqiao@apache.org>
This commit is contained in:
gavin.wang 2024-07-16 20:57:50 +08:00 committed by GitHub
parent 51cb858cc8
commit 5730656660
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 22 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View File

@ -587,7 +587,7 @@ void parseArguments(List<String> args) {
numOpsRequired = Integer.parseInt(args.get(++i));
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Long.parseLong(args.get(++i));
blockSize = StringUtils.TraditionalBinaryPrefix.string2long(args.get(++i));
} else if(args.get(i).equals("-threads")) {
if(i+1 == args.size()) printUsage();
numThreads = Integer.parseInt(args.get(++i));
@ -1260,7 +1260,7 @@ void parseArguments(List<String> args) {
blocksPerFile = Integer.parseInt(args.get(++i));
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Long.parseLong(args.get(++i));
blockSize = StringUtils.TraditionalBinaryPrefix.string2long(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if (i + 1 == args.size()) {
printUsage();
@ -1498,7 +1498,7 @@ void parseArguments(List<String> args) {
replication = Short.parseShort(args.get(++i));
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Long.parseLong(args.get(++i));
blockSize = StringUtils.TraditionalBinaryPrefix.string2long(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if (i + 1 == args.size()) {
printUsage();

19
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
View File

@ -285,4 +285,23 @@ public void testNNThroughputWithBlockSize() throws Exception {
"-blockSize", "32", "-close"});
}
}
/**
* This test runs {@link NNThroughputBenchmark} against a mini DFS cluster
* with explicit -blockSize option like 1m.
*/
@Test(timeout = 120000)
public void testNNThroughputBlockSizeArgWithLetterSuffix() throws Exception {
final Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
cluster.waitActive();
final Configuration benchConf = new HdfsConfiguration();
benchConf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
FileSystem.setDefaultUri(benchConf, cluster.getURI());
NNThroughputBenchmark.runBenchmark(benchConf,
new String[]{"-op", "create", "-keepResults", "-files", "3",
"-blockSize", "1m", "-close"});
}
}
}