big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-10328. loadGenerator exit code is not reliable. Contributed by Haohui Mai.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1570304 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2014-02-20 18:58:10 +00:00
parent 5a3460cf78
commit 835484c88f
2 changed files with 75 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -393,6 +393,9 @@ Release 2.4.0 - UNRELEASED
HADOOP-10346. Deadlock while logging tokens (jlowe)
HADOOP-10328. loadGenerator exit code is not reliable.
(Haohui Mai via cnauroth)
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES

104
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
View File

@ -26,6 +26,7 @@
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.Random;
@ -39,6 +40,7 @@
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@ -137,11 +139,15 @@ public class LoadGenerator extends Configured implements Tool {
"-startTime <startTimeInMillis>\n" +
"-scriptFile <filename>";
final private String hostname;
private final byte[] WRITE_CONTENTS = new byte[4096];
private static final int ERR_TEST_FAILED = 2;
/** Constructor */
public LoadGenerator() throws IOException, UnknownHostException {
InetAddress addr = InetAddress.getLocalHost();
hostname = addr.getHostName();
Arrays.fill(WRITE_CONTENTS, (byte) 'a');
}
private final static int OPEN = 0;
@ -178,6 +184,7 @@ private class DFSClientThread extends Thread {
private long [] executionTime = new long[TOTAL_OP_TYPES];
private long [] totalNumOfOps = new long[TOTAL_OP_TYPES];
private byte[] buffer = new byte[1024];
private boolean failed;
private DFSClientThread(int id) {
this.id = id;
@ -196,6 +203,7 @@ public void run() {
} catch (Exception ioe) {
System.err.println(ioe.getLocalizedMessage());
ioe.printStackTrace();
failed = true;
}
}
@ -272,6 +280,35 @@ private void list() throws IOException {
executionTime[LIST] += (Time.now()-startTime);
totalNumOfOps[LIST]++;
}
/** Create a file with a length of <code>fileSize</code>.
* The file is filled with 'a'.
*/
private void genFile(Path file, long fileSize) throws IOException {
long startTime = Time.now();
FSDataOutputStream out = null;
try {
out = fc.create(file,
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
CreateOpts.createParent(), CreateOpts.bufferSize(4096),
CreateOpts.repFac((short) 3));
executionTime[CREATE] += (Time.now() - startTime);
totalNumOfOps[CREATE]++;
long i = fileSize;
while (i > 0) {
long s = Math.min(fileSize, WRITE_CONTENTS.length);
out.write(WRITE_CONTENTS, 0, (int) s);
i -= s;
}
startTime = Time.now();
executionTime[WRITE_CLOSE] += (Time.now() - startTime);
totalNumOfOps[WRITE_CLOSE]++;
} finally {
IOUtils.cleanup(LOG, out);
}
}
}
/** Main function:
@ -319,13 +356,21 @@ public int run(String[] args) throws Exception {
if(LOG.isDebugEnabled()) {
LOG.debug("Done with testing. Waiting for threads to finish.");
}
boolean failed = false;
for (DFSClientThread thread : threads) {
thread.join();
for (int i=0; i<TOTAL_OP_TYPES; i++) {
executionTime[i] += thread.executionTime[i];
totalNumOfOps[i] += thread.totalNumOfOps[i];
}
failed = failed || thread.failed;
}
if (failed) {
exitCode = -ERR_TEST_FAILED;
}
long totalOps = 0;
for (int i=0; i<TOTAL_OP_TYPES; i++) {
totalOps += totalNumOfOps[i];
@ -462,50 +507,50 @@ private int loadScriptFile(String filename) throws IOException {
String line;
// Read script, parse values, build array of duration, read and write probs
while((line = br.readLine()) != null) {
while ((line = br.readLine()) != null) {
lineNum++;
if(line.startsWith("#") || line.isEmpty()) // skip comments and blanks
if (line.startsWith("#") || line.isEmpty()) // skip comments and blanks
continue;
String[] a = line.split("\\s");
if(a.length != 3) {
System.err.println("Line " + lineNum +
": Incorrect number of parameters: " + line);
if (a.length != 3) {
System.err.println("Line " + lineNum
+ ": Incorrect number of parameters: " + line);
}
try {
long d = Long.parseLong(a[0]);
if(d < 0) {
System.err.println("Line " + lineNum + ": Invalid duration: " + d);
return -1;
if (d < 0) {
System.err.println("Line " + lineNum + ": Invalid duration: " + d);
return -1;
}
double r = Double.parseDouble(a[1]);
if(r < 0.0 || r > 1.0 ) {
System.err.println("Line " + lineNum +
": The read probability must be [0, 1]: " + r);
return -1;
if (r < 0.0 || r > 1.0) {
System.err.println("Line " + lineNum
+ ": The read probability must be [0, 1]: " + r);
return -1;
}
double w = Double.parseDouble(a[2]);
if(w < 0.0 || w > 1.0) {
System.err.println("Line " + lineNum +
": The read probability must be [0, 1]: " + r);
if (w < 0.0 || w > 1.0) {
System.err.println("Line " + lineNum
+ ": The read probability must be [0, 1]: " + r);
return -1;
}
readProb.add(r);
duration.add(d);
writeProb.add(w);
} catch( NumberFormatException nfe) {
} catch (NumberFormatException nfe) {
System.err.println(lineNum + ": Can't parse: " + line);
return -1;
} finally {
IOUtils.cleanup(LOG, br);
}
}
br.close();
fr.close();
// Copy vectors to arrays of values, to avoid autoboxing overhead later
durations = new long[duration.size()];
readProbs = new double[readProb.size()];
@ -582,27 +627,6 @@ private void barrier() {
}
}
/** Create a file with a length of <code>fileSize</code>.
* The file is filled with 'a'.
*/
private void genFile(Path file, long fileSize) throws IOException {
long startTime = Time.now();
FSDataOutputStream out = fc.create(file,
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
CreateOpts.createParent(), CreateOpts.bufferSize(4096),
CreateOpts.repFac((short) 3));
executionTime[CREATE] += (Time.now()-startTime);
totalNumOfOps[CREATE]++;
for (long i=0; i<fileSize; i++) {
out.writeByte('a');
}
startTime = Time.now();
out.close();
executionTime[WRITE_CLOSE] += (Time.now()-startTime);
totalNumOfOps[WRITE_CLOSE]++;
}
/** Main program
*
* @param args command line arguments