HDFS-17272. NNThroughputBenchmark should support specifying the base directory for multi-client test (#6319). Contributed by caozhiqiang.
Reviewed-by: Tao Li <tomscut@apache.org> Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
caozhiqiang
GitHub
parent
f96361fd54
commit
37d6cada14
@ -54,15 +54,15 @@ Following are all the operations supported along with their respective operation
|
||||
| OPERATION\_OPTION | Operation-specific parameters |
|
||||
|:---- |:---- |
|
||||
|`all` | _options for other operations_ |
|
||||
|`create` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-close`] |
|
||||
|`mkdirs` | [`-threads 3`] [`-dirs 10`] [`-dirsPerDir 2`] |
|
||||
|`open` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|
||||
|`delete` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|
||||
|`append` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-appendNewBlk`] |
|
||||
|`fileStatus` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|
||||
|`rename` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|
||||
|`blockReport` | [`-datanodes 10`] [`-reports 30`] [`-blocksPerReport 100`] [`-blocksPerFile 10`] |
|
||||
|`replication` | [`-datanodes 10`] [`-nodesToDecommission 1`] [`-nodeReplicationLimit 100`] [`-totalBlocks 100`] [`-replication 3`] |
|
||||
|`create` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-close`] [`-baseDirName /nnThroughputBenchmark`] |
|
||||
|`mkdirs` | [`-threads 3`] [`-dirs 10`] [`-dirsPerDir 2`] [`-baseDirName /nnThroughputBenchmark`] |
|
||||
|`open` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|
||||
|`delete` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|
||||
|`append` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-appendNewBlk`] [`-baseDirName /nnThroughputBenchmark`] |
|
||||
|`fileStatus` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|
||||
|`rename` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|
||||
|`blockReport` | [`-datanodes 10`] [`-reports 30`] [`-blocksPerReport 100`] [`-blocksPerFile 10`] [`-baseDirName /nnThroughputBenchmark`] |
|
||||
|`replication` | [`-datanodes 10`] [`-nodesToDecommission 1`] [`-nodeReplicationLimit 100`] [`-totalBlocks 100`] [`-replication 3`] [`-baseDirName /nnThroughputBenchmark`] |
|
||||
|`clean` | N/A |
|
||||
|
||||
##### Operation Options
|
||||
@ -86,6 +86,7 @@ When running benchmarks with the above operation(s), please provide operation-sp
|
||||
|`-nodeReplicationLimit` | The maximum number of outgoing replication streams for a data-node. |
|
||||
|`-totalBlocks` | Number of total blocks to operate. |
|
||||
|`-replication` | Replication factor. Will be adjusted to number of data-nodes if it is larger than that. |
|
||||
|`-baseDirName` | The base dir name for benchmarks, to support multiple clients submitting benchmark tests at the same time. |
|
||||
|
||||
### Reports
|
||||
|
||||
|
||||
@ -162,11 +162,11 @@ static void setNameNodeLoggingLevel(Level logLevel) {
|
||||
* specific name-node operation.
|
||||
*/
|
||||
abstract class OperationStatsBase {
|
||||
protected static final String BASE_DIR_NAME = "/nnThroughputBenchmark";
|
||||
private String baseDirName = "/nnThroughputBenchmark";
|
||||
protected static final String OP_ALL_NAME = "all";
|
||||
protected static final String OP_ALL_USAGE = "-op all <other ops options>";
|
||||
|
||||
protected final String baseDir;
|
||||
private String baseDir;
|
||||
protected short replication;
|
||||
protected int blockSize;
|
||||
protected int numThreads = 0; // number of threads
|
||||
@ -228,7 +228,7 @@ abstract class OperationStatsBase {
|
||||
abstract void printResults();
|
||||
|
||||
OperationStatsBase() {
|
||||
baseDir = BASE_DIR_NAME + "/" + getOpName();
|
||||
baseDir = baseDirName + "/" + getOpName();
|
||||
replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
|
||||
blockSize = config.getInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
|
||||
numOpsRequired = 10;
|
||||
@ -295,7 +295,12 @@ void cleanUp() throws IOException {
|
||||
clientProto.saveNamespace(0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
public String getBaseDirName() {
|
||||
return baseDirName;
|
||||
}
|
||||
public void setBaseDirName(String baseDirName) {
|
||||
this.baseDirName = baseDirName;
|
||||
}
|
||||
int getNumOpsExecuted() {
|
||||
return numOpsExecuted;
|
||||
}
|
||||
@ -316,10 +321,15 @@ long getAverageTime() {
|
||||
return elapsedTime == 0 ? 0 : 1000*(double)numOpsExecuted / elapsedTime;
|
||||
}
|
||||
|
||||
String getBaseDir() {
|
||||
public String getBaseDir() {
|
||||
setBaseDir(baseDirName + "/" + getOpName());
|
||||
return baseDir;
|
||||
}
|
||||
|
||||
public void setBaseDir(String baseDir) {
|
||||
this.baseDir = baseDir;
|
||||
}
|
||||
|
||||
String getClientName(int idx) {
|
||||
return getOpName() + "-client-" + idx;
|
||||
}
|
||||
@ -494,7 +504,7 @@ long executeOp(int daemonId, int inputIdx, String ignore)
|
||||
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
|
||||
false);
|
||||
long start = Time.now();
|
||||
clientProto.delete(BASE_DIR_NAME, true);
|
||||
clientProto.delete(getBaseDirName(), true);
|
||||
long end = Time.now();
|
||||
return end-start;
|
||||
}
|
||||
@ -502,7 +512,7 @@ long executeOp(int daemonId, int inputIdx, String ignore)
|
||||
@Override
|
||||
void printResults() {
|
||||
LOG.info("--- " + getOpName() + " inputs ---");
|
||||
LOG.info("Remove directory " + BASE_DIR_NAME);
|
||||
LOG.info("Remove directory " + getBaseDirName());
|
||||
printStats();
|
||||
}
|
||||
}
|
||||
@ -519,7 +529,7 @@ class CreateFileStats extends OperationStatsBase {
|
||||
static final String OP_CREATE_NAME = "create";
|
||||
static final String OP_CREATE_USAGE =
|
||||
"-op create [-threads T] [-files N] [-blockSize S] [-filesPerDir P]"
|
||||
+ " [-close]";
|
||||
+ " [-baseDirName D] [-close]";
|
||||
|
||||
protected FileNameGenerator nameGenerator;
|
||||
protected String[][] fileNames;
|
||||
@ -553,6 +563,11 @@ void parseArguments(List<String> args) {
|
||||
} else if(args.get(i).equals("-filesPerDir")) {
|
||||
if(i+1 == args.size()) printUsage();
|
||||
nrFilesPerDir = Integer.parseInt(args.get(++i));
|
||||
} else if(args.get(i).equals("-baseDirName")) {
|
||||
if (i + 1 == args.size()) {
|
||||
printUsage();
|
||||
}
|
||||
setBaseDirName(args.get(++i));
|
||||
} else if(args.get(i).equals("-close")) {
|
||||
closeUponCreate = true;
|
||||
} else if(!ignoreUnrelatedOptions)
|
||||
@ -568,6 +583,7 @@ void generateInputs(int[] opsPerThread) throws IOException {
|
||||
false);
|
||||
// int generatedFileIdx = 0;
|
||||
LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName());
|
||||
LOG.info("basedir: " + getBaseDir());
|
||||
fileNames = new String[numThreads][];
|
||||
try {
|
||||
for(int idx=0; idx < numThreads; idx++) {
|
||||
@ -618,6 +634,7 @@ long executeOp(int daemonId, int inputIdx, String clientName)
|
||||
@Override
|
||||
void printResults() {
|
||||
LOG.info("--- " + getOpName() + " inputs ---");
|
||||
LOG.info("baseDir = " + getBaseDir());
|
||||
LOG.info("nrFiles = " + numOpsRequired);
|
||||
LOG.info("nrThreads = " + numThreads);
|
||||
LOG.info("nrFilesPerDir = " + nameGenerator.getFilesPerDirectory());
|
||||
@ -635,7 +652,7 @@ class MkdirsStats extends OperationStatsBase {
|
||||
// Operation types
|
||||
static final String OP_MKDIRS_NAME = "mkdirs";
|
||||
static final String OP_MKDIRS_USAGE = "-op mkdirs [-threads T] [-dirs N] " +
|
||||
"[-dirsPerDir P]";
|
||||
"[-dirsPerDir P] [-baseDirName D]";
|
||||
|
||||
protected FileNameGenerator nameGenerator;
|
||||
protected String[][] dirPaths;
|
||||
@ -664,6 +681,11 @@ void parseArguments(List<String> args) {
|
||||
} else if(args.get(i).equals("-dirsPerDir")) {
|
||||
if(i+1 == args.size()) printUsage();
|
||||
nrDirsPerDir = Integer.parseInt(args.get(++i));
|
||||
} else if(args.get(i).equals("-baseDirName")) {
|
||||
if (i + 1 == args.size()) {
|
||||
printUsage();
|
||||
}
|
||||
setBaseDirName(args.get(++i));
|
||||
} else if(!ignoreUnrelatedOptions)
|
||||
printUsage();
|
||||
}
|
||||
@ -718,6 +740,7 @@ long executeOp(int daemonId, int inputIdx, String clientName)
|
||||
@Override
|
||||
void printResults() {
|
||||
LOG.info("--- " + getOpName() + " inputs ---");
|
||||
LOG.info("baseDir = " + getBaseDir());
|
||||
LOG.info("nrDirs = " + numOpsRequired);
|
||||
LOG.info("nrThreads = " + numThreads);
|
||||
LOG.info("nrDirsPerDir = " + nameGenerator.getFilesPerDirectory());
|
||||
@ -736,7 +759,7 @@ class OpenFileStats extends CreateFileStats {
|
||||
static final String OP_OPEN_NAME = "open";
|
||||
static final String OP_USAGE_ARGS =
|
||||
" [-threads T] [-files N] [-blockSize S] [-filesPerDir P]"
|
||||
+ " [-useExisting]";
|
||||
+ " [-useExisting] [-baseDirName D]";
|
||||
static final String OP_OPEN_USAGE =
|
||||
"-op " + OP_OPEN_NAME + OP_USAGE_ARGS;
|
||||
|
||||
@ -771,6 +794,7 @@ void generateInputs(int[] opsPerThread) throws IOException {
|
||||
"-blockSize", String.valueOf(blockSize),
|
||||
"-filesPerDir",
|
||||
String.valueOf(nameGenerator.getFilesPerDirectory()),
|
||||
"-baseDirName", getBaseDirName(),
|
||||
"-close"};
|
||||
CreateFileStats opCreate = new CreateFileStats(Arrays.asList(createArgs));
|
||||
|
||||
@ -1137,7 +1161,7 @@ class BlockReportStats extends OperationStatsBase {
|
||||
static final String OP_BLOCK_REPORT_NAME = "blockReport";
|
||||
static final String OP_BLOCK_REPORT_USAGE =
|
||||
"-op blockReport [-datanodes T] [-reports N] " +
|
||||
"[-blocksPerReport B] [-blocksPerFile F] [-blockSize S]";
|
||||
"[-blocksPerReport B] [-blocksPerFile F] [-blockSize S] [-baseDirName D]";
|
||||
|
||||
private int blocksPerReport;
|
||||
private int blocksPerFile;
|
||||
@ -1187,6 +1211,11 @@ void parseArguments(List<String> args) {
|
||||
} else if (args.get(i).equals("-blockSize")) {
|
||||
if(i+1 == args.size()) printUsage();
|
||||
blockSize = Integer.parseInt(args.get(++i));
|
||||
} else if(args.get(i).equals("-baseDirName")) {
|
||||
if (i + 1 == args.size()) {
|
||||
printUsage();
|
||||
}
|
||||
setBaseDirName(args.get(++i));
|
||||
} else if(!ignoreUnrelatedOptions)
|
||||
printUsage();
|
||||
}
|
||||
@ -1330,6 +1359,7 @@ void printResults() {
|
||||
}
|
||||
blockDistribution += ")";
|
||||
LOG.info("--- " + getOpName() + " inputs ---");
|
||||
LOG.info("baseDir = " + getBaseDir());
|
||||
LOG.info("reports = " + numOpsRequired);
|
||||
LOG.info("datanodes = " + numThreads + " " + blockDistribution);
|
||||
LOG.info("blocksPerReport = " + blocksPerReport);
|
||||
@ -1348,7 +1378,7 @@ class ReplicationStats extends OperationStatsBase {
|
||||
static final String OP_REPLICATION_USAGE =
|
||||
"-op replication [-datanodes T] [-nodesToDecommission D] " +
|
||||
"[-nodeReplicationLimit C] [-totalBlocks B] [-blockSize S] "
|
||||
+ "[-replication R]";
|
||||
+ "[-replication R] [-baseDirName D]";
|
||||
|
||||
private final BlockReportStats blockReportObject;
|
||||
private int numDatanodes;
|
||||
@ -1377,7 +1407,8 @@ class ReplicationStats extends OperationStatsBase {
|
||||
"-datanodes", String.valueOf(numDatanodes),
|
||||
"-blocksPerReport", String.valueOf(totalBlocks*replication/numDatanodes),
|
||||
"-blocksPerFile", String.valueOf(numDatanodes),
|
||||
"-blockSize", String.valueOf(blockSize)};
|
||||
"-blockSize", String.valueOf(blockSize),
|
||||
"-baseDirName", getBaseDirName()};
|
||||
blockReportObject = new BlockReportStats(Arrays.asList(blkReportArgs));
|
||||
numDecommissionedBlocks = 0;
|
||||
numPendingBlocks = 0;
|
||||
@ -1410,6 +1441,11 @@ void parseArguments(List<String> args) {
|
||||
} else if (args.get(i).equals("-blockSize")) {
|
||||
if(i+1 == args.size()) printUsage();
|
||||
blockSize = Integer.parseInt(args.get(++i));
|
||||
} else if(args.get(i).equals("-baseDirName")) {
|
||||
if (i + 1 == args.size()) {
|
||||
printUsage();
|
||||
}
|
||||
setBaseDirName(args.get(++i));
|
||||
} else if(!ignoreUnrelatedOptions)
|
||||
printUsage();
|
||||
}
|
||||
@ -1485,6 +1521,7 @@ void printResults() {
|
||||
}
|
||||
blockDistribution += ")";
|
||||
LOG.info("--- " + getOpName() + " inputs ---");
|
||||
LOG.info("baseDir = " + getBaseDir());
|
||||
LOG.info("numOpsRequired = " + numOpsRequired);
|
||||
LOG.info("datanodes = " + numDatanodes + " " + blockDistribution);
|
||||
LOG.info("decommissioned datanodes = " + nodesToDecommission);
|
||||
@ -1631,7 +1668,7 @@ public int run(String[] aArgs) throws Exception {
|
||||
}
|
||||
// run each benchmark
|
||||
for(OperationStatsBase op : ops) {
|
||||
LOG.info("Starting benchmark: " + op.getOpName());
|
||||
LOG.info("Starting benchmark: " + op.getOpName() + ", baseDir: " + op.getBaseDir());
|
||||
op.benchmark();
|
||||
op.cleanUp();
|
||||
}
|
||||
|
||||
@ -22,8 +22,10 @@
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||
@ -187,4 +189,38 @@ public void testNNThroughputForBlockReportOp() throws Exception {
|
||||
"blockReport", "-datanodes", "3", "-reports", "2"});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This test runs {@link NNThroughputBenchmark} against a mini DFS cluster
|
||||
* with explicit -baseDirName option.
|
||||
*/
|
||||
@Test(timeout = 120000)
|
||||
public void testNNThroughputWithBaseDir() throws Exception {
|
||||
final Configuration conf = new HdfsConfiguration();
|
||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
|
||||
MiniDFSCluster cluster = null;
|
||||
try {
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
|
||||
cluster.waitActive();
|
||||
final Configuration benchConf = new HdfsConfiguration();
|
||||
benchConf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
|
||||
FileSystem.setDefaultUri(benchConf, cluster.getURI());
|
||||
DistributedFileSystem fs = cluster.getFileSystem();
|
||||
|
||||
NNThroughputBenchmark.runBenchmark(benchConf,
|
||||
new String[] {"-op", "create", "-keepResults", "-files", "3", "-baseDirName",
|
||||
"/nnThroughputBenchmark1", "-close"});
|
||||
Assert.assertTrue(fs.exists(new Path("/nnThroughputBenchmark1")));
|
||||
Assert.assertFalse(fs.exists(new Path("/nnThroughputBenchmark")));
|
||||
|
||||
NNThroughputBenchmark.runBenchmark(benchConf,
|
||||
new String[] {"-op", "all", "-baseDirName", "/nnThroughputBenchmark1"});
|
||||
Assert.assertTrue(fs.exists(new Path("/nnThroughputBenchmark1")));
|
||||
Assert.assertFalse(fs.exists(new Path("/nnThroughputBenchmark")));
|
||||
} finally {
|
||||
if (cluster != null) {
|
||||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user