big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6729. Accurately compute the test execute time in DFSIO. Contributed by mingleizhang.

Browse Source 
This closes #112
This commit is contained in:
Akira Ajisaka 2016-07-30 22:13:19 +09:00
parent 8d32bd884d
commit ce93595d7a
1 changed files with 27 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
View File

@ -228,57 +228,45 @@ public static void afterClass() throws Exception {
public static void testWrite() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.writeTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.writeTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_WRITE, execTime);
}
@Test (timeout = 10000)
public void testRead() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.readTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.readTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_READ, execTime);
}
@Test (timeout = 10000)
public void testReadRandom() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.getConf().setLong("test.io.skip.size", 0);
bench.randomReadTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.randomReadTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_RANDOM, execTime);
}
@Test (timeout = 10000)
public void testReadBackward() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.getConf().setLong("test.io.skip.size", -DEFAULT_BUFFER_SIZE);
bench.randomReadTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.randomReadTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_BACKWARD, execTime);
}
@Test (timeout = 10000)
public void testReadSkip() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.getConf().setLong("test.io.skip.size", 1);
bench.randomReadTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.randomReadTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_SKIP, execTime);
}
@Test (timeout = 10000)
public void testAppend() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.appendTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.appendTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_APPEND, execTime);
}
@ -286,9 +274,7 @@ public void testAppend() throws Exception {
public void testTruncate() throws Exception {
FileSystem fs = cluster.getFileSystem();
bench.createControlFile(fs, DEFAULT_NR_BYTES / 2, DEFAULT_NR_FILES);
long tStart = System.currentTimeMillis();
bench.truncateTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.truncateTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_TRUNCATE, execTime);
}
@ -430,12 +416,14 @@ public Long doIO(Reporter reporter,
}
}
private void writeTest(FileSystem fs) throws IOException {
private long writeTest(FileSystem fs) throws IOException {
Path writeDir = getWriteDir(config);
fs.delete(getDataDir(config), true);
fs.delete(writeDir, true);
long tStart = System.currentTimeMillis();
runIOTest(WriteMapper.class, writeDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}
private void runIOTest(
@ -496,10 +484,13 @@ public Long doIO(Reporter reporter,
}
}
private void appendTest(FileSystem fs) throws IOException {
private long appendTest(FileSystem fs) throws IOException {
Path appendDir = getAppendDir(config);
fs.delete(appendDir, true);
long tStart = System.currentTimeMillis();
runIOTest(AppendMapper.class, appendDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}
/**
@ -539,10 +530,13 @@ public Long doIO(Reporter reporter,
}
}
private void readTest(FileSystem fs) throws IOException {
private long readTest(FileSystem fs) throws IOException {
Path readDir = getReadDir(config);
fs.delete(readDir, true);
long tStart = System.currentTimeMillis();
runIOTest(ReadMapper.class, readDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}
/**
@ -620,10 +614,13 @@ private long nextOffset(long current) {
}
}
private void randomReadTest(FileSystem fs) throws IOException {
private long randomReadTest(FileSystem fs) throws IOException {
Path readDir = getRandomReadDir(config);
fs.delete(readDir, true);
long tStart = System.currentTimeMillis();
runIOTest(RandomReadMapper.class, readDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}
/**
@ -665,10 +662,13 @@ public Long doIO(Reporter reporter,
}
}
private void truncateTest(FileSystem fs) throws IOException {
private long truncateTest(FileSystem fs) throws IOException {
Path TruncateDir = getTruncateDir(config);
fs.delete(TruncateDir, true);
long tStart = System.currentTimeMillis();
runIOTest(TruncateMapper.class, TruncateDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}
private void sequentialTest(FileSystem fs,