big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16389.Improve NNThroughputBenchmark test mkdirs. (#3819)

Browse Source 
Reviewed-by: Viraj Jasani <vjasani@apache.org>
Reviewed-by: Akira Ajisaka <aajisaka@apache.org>
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
jianghuazhu 2022-04-18 11:21:51 +08:00 committed by GitHub
parent 35d4c02bcc
commit 900682e712
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 32 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
View File

@ -77,6 +77,10 @@ synchronized String getNextFileName(String fileNamePrefix) {
return fn; return fn;
} }
public long getFileCount() {
return fileCount;
}
private synchronized void reset() { private synchronized void reset() {
Arrays.fill(pathIndecies, -1); Arrays.fill(pathIndecies, -1);
fileCount = 0L; fileCount = 0L;

40
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View File

@ -569,12 +569,20 @@ void generateInputs(int[] opsPerThread) throws IOException {
// int generatedFileIdx = 0; // int generatedFileIdx = 0;
LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName()); LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName());
fileNames = new String[numThreads][]; fileNames = new String[numThreads][];
for(int idx=0; idx < numThreads; idx++) { try {
int threadOps = opsPerThread[idx]; for(int idx=0; idx < numThreads; idx++) {
fileNames[idx] = new String[threadOps]; int threadOps = opsPerThread[idx];
for(int jdx=0; jdx < threadOps; jdx++) fileNames[idx] = new String[threadOps];
fileNames[idx][jdx] = nameGenerator. for(int jdx=0; jdx < threadOps; jdx++) {
getNextFileName("ThroughputBench"); fileNames[idx][jdx] = nameGenerator.
getNextFileName("ThroughputBench");
}
}
} catch (ArrayIndexOutOfBoundsException e) {
LOG.error("The current environment allows {} files to be created. " +
"If you want to test more files, please update the -filesPerDir parameter.",
nameGenerator.getFileCount());
throw e;
} }
} }
@ -669,12 +677,20 @@ void generateInputs(int[] opsPerThread) throws IOException {
false); false);
LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName()); LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName());
dirPaths = new String[numThreads][]; dirPaths = new String[numThreads][];
for(int idx=0; idx < numThreads; idx++) { try {
int threadOps = opsPerThread[idx]; for(int idx=0; idx < numThreads; idx++) {
dirPaths[idx] = new String[threadOps]; int threadOps = opsPerThread[idx];
for(int jdx=0; jdx < threadOps; jdx++) dirPaths[idx] = new String[threadOps];
dirPaths[idx][jdx] = nameGenerator. for(int jdx=0; jdx < threadOps; jdx++) {
getNextFileName("ThroughputBench"); dirPaths[idx][jdx] = nameGenerator.
getNextFileName("ThroughputBench");
}
}
} catch (ArrayIndexOutOfBoundsException e) {
LOG.error("The current environment allows {} directories to be created. " +
"If you want to test more directories, please update the -dirsPerDir parameter.",
nameGenerator.getFileCount());
throw e;
} }
} }