big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5248. Let NNBenchWithoutMR specify the replication factor for its test. Contributed by Erik Paulson

Browse Source
This commit is contained in:
Jason Lowe 2015-05-08 22:28:50 +00:00
parent c945c20483
commit 30099a36c6
2 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -322,6 +322,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-5981. Log levels of certain MR logs can be changed to DEBUG. MAPREDUCE-5981. Log levels of certain MR logs can be changed to DEBUG.
(Varun Saxena via devaraj) (Varun Saxena via devaraj)
MAPREDUCE-5248. Let NNBenchWithoutMR specify the replication factor for
its test (Erik Paulson via jlowe)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

23
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java
View File

@ -54,6 +54,7 @@ public class NNBenchWithoutMR {
private static long bytesPerBlock = 1; private static long bytesPerBlock = 1;
private static long blocksPerFile = 0; private static long blocksPerFile = 0;
private static long bytesPerFile = 1; private static long bytesPerFile = 1;
private static short replicationFactorPerFile = 1; // default is 1
private static Path baseDir = null; private static Path baseDir = null;
// variables initialized in main() // variables initialized in main()
@ -106,7 +107,7 @@ static int createWrite() {
try { try {
out = fileSys.create( out = fileSys.create(
new Path(taskDir, "" + index), false, 512, new Path(taskDir, "" + index), false, 512,
(short)1, bytesPerBlock); (short)replicationFactorPerFile, bytesPerBlock);
success = true; success = true;
} catch (IOException ioe) { } catch (IOException ioe) {
success=false; success=false;
@ -263,14 +264,15 @@ public static void main(String[] args) throws IOException {
String usage = String usage =
"Usage: nnbench " + "Usage: nnbench " +
" -operation <one of createWrite, openRead, rename, or delete> " + " -operation <one of createWrite, openRead, rename, or delete>\n " +
" -baseDir <base output/input DFS path> " + " -baseDir <base output/input DFS path>\n " +
" -startTime <time to start, given in seconds from the epoch> " + " -startTime <time to start, given in seconds from the epoch>\n" +
" -numFiles <number of files to create> " + " -numFiles <number of files to create>\n " +
" -blocksPerFile <number of blocks to create per file> " + " -replicationFactorPerFile <Replication factor for the files, default is 1>\n" +
" [-bytesPerBlock <number of bytes to write to each block, default is 1>] " + " -blocksPerFile <number of blocks to create per file>\n" +
" [-bytesPerChecksum <value for io.bytes.per.checksum>]" + " [-bytesPerBlock <number of bytes to write to each block, default is 1>]\n" +
"Note: bytesPerBlock MUST be a multiple of bytesPerChecksum"; " [-bytesPerChecksum <value for io.bytes.per.checksum>]\n" +
"Note: bytesPerBlock MUST be a multiple of bytesPerChecksum\n";
String operation = null; String operation = null;
for (int i = 0; i < args.length; i++) { // parse command line for (int i = 0; i < args.length; i++) { // parse command line
@ -284,6 +286,8 @@ public static void main(String[] args) throws IOException {
bytesPerBlock = Long.parseLong(args[++i]); bytesPerBlock = Long.parseLong(args[++i]);
} else if (args[i].equals("-bytesPerChecksum")) { } else if (args[i].equals("-bytesPerChecksum")) {
bytesPerChecksum = Integer.parseInt(args[++i]); bytesPerChecksum = Integer.parseInt(args[++i]);
} else if (args[i].equals("-replicationFactorPerFile")) {
replicationFactorPerFile = Short.parseShort(args[++i]);
} else if (args[i].equals("-startTime")) { } else if (args[i].equals("-startTime")) {
startTime = Long.parseLong(args[++i]) * 1000; startTime = Long.parseLong(args[++i]) * 1000;
} else if (args[i].equals("-operation")) { } else if (args[i].equals("-operation")) {
@ -307,6 +311,7 @@ public static void main(String[] args) throws IOException {
System.out.println(" baseDir: " + baseDir); System.out.println(" baseDir: " + baseDir);
System.out.println(" startTime: " + startTime); System.out.println(" startTime: " + startTime);
System.out.println(" numFiles: " + numFiles); System.out.println(" numFiles: " + numFiles);
System.out.println(" replicationFactorPerFile: " + replicationFactorPerFile);
System.out.println(" blocksPerFile: " + blocksPerFile); System.out.println(" blocksPerFile: " + blocksPerFile);
System.out.println(" bytesPerBlock: " + bytesPerBlock); System.out.println(" bytesPerBlock: " + bytesPerBlock);
System.out.println(" bytesPerChecksum: " + bytesPerChecksum); System.out.println(" bytesPerChecksum: " + bytesPerChecksum);