big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-4110. Fix tests in TestMiniMRClasspath & TestMiniMRWithDFSWithDistinctUsers (Devaraj K via bobby)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1310546 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-04-06 18:56:55 +00:00
parent 5af7077dcd
commit b435e2202d
3 changed files with 58 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-mapreduce-project/CHANGES.txt
View File

@ -213,7 +213,11 @@ Release 2.0.0 - UNRELEASED
MAPREDUCE-4111. Fix tests in org.apache.hadoop.mapred.TestJobName (Devaraj
K via bobby)
MAPREDUCE-4110. Fix tests in org.apache.hadoop.mapred.TestMiniMRClasspath &
org.apache.hadoop.mapred.TestMiniMRWithDFSWithDistinctUsers (Devaraj K via
bobby)
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES

77
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
View File

@ -18,47 +18,43 @@
package org.apache.hadoop.mapred;
import java.io.*;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.junit.Ignore;
import org.junit.Assert;
import org.junit.Test;
/**
* A JUnit test to test Mini Map-Reduce Cluster with multiple directories
* and check for correct classpath
*/
@Ignore
public class TestMiniMRClasspath extends TestCase {
public class TestMiniMRClasspath {
static void configureWordCount(FileSystem fs,
String jobTracker,
JobConf conf,
String input,
int numMaps,
int numReduces,
Path inDir, Path outDir) throws IOException {
static void configureWordCount(FileSystem fs, JobConf conf, String input,
int numMaps, int numReduces, Path inDir, Path outDir) throws IOException {
fs.delete(outDir, true);
if (!fs.mkdirs(inDir)) {
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
{
DataOutputStream file = fs.create(new Path(inDir, "part-0"));
file.writeBytes(input);
file.close();
}
DataOutputStream file = fs.create(new Path(inDir, "part-0"));
file.writeBytes(input);
file.close();
FileSystem.setDefaultUri(conf, fs.getUri());
conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
conf.setJobName("wordcount");
conf.setInputFormat(TextInputFormat.class);
@ -74,18 +70,17 @@ static void configureWordCount(FileSystem fs,
FileOutputFormat.setOutputPath(conf, outDir);
conf.setNumMapTasks(numMaps);
conf.setNumReduceTasks(numReduces);
//pass a job.jar already included in the hadoop build
conf.setJar("build/test/mapred/testjar/testjob.jar");
//set the tests jar file
conf.setJarByClass(TestMiniMRClasspath.class);
}
static String launchWordCount(URI fileSys, String jobTracker, JobConf conf,
String input, int numMaps, int numReduces)
static String launchWordCount(URI fileSys, JobConf conf, String input,
int numMaps, int numReduces)
throws IOException {
final Path inDir = new Path("/testing/wc/input");
final Path outDir = new Path("/testing/wc/output");
FileSystem fs = FileSystem.get(fileSys, conf);
configureWordCount(fs, jobTracker, conf, input, numMaps, numReduces, inDir,
outDir);
configureWordCount(fs, conf, input, numMaps, numReduces, inDir, outDir);
JobClient.runJob(conf);
StringBuffer result = new StringBuffer();
{
@ -107,8 +102,8 @@ static String launchWordCount(URI fileSys, String jobTracker, JobConf conf,
return result.toString();
}
static String launchExternal(URI uri, String jobTracker, JobConf conf,
String input, int numMaps, int numReduces)
static String launchExternal(URI uri, JobConf conf, String input,
int numMaps, int numReduces)
throws IOException {
final Path inDir = new Path("/testing/ext/input");
@ -124,8 +119,7 @@ static String launchExternal(URI uri, String jobTracker, JobConf conf,
file.close();
}
FileSystem.setDefaultUri(conf, uri);
conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
conf.setJobName("wordcount");
conf.setInputFormat(TextInputFormat.class);
@ -142,8 +136,8 @@ static String launchExternal(URI uri, String jobTracker, JobConf conf,
conf.set("mapred.mapper.class", "testjar.ExternalMapperReducer");
conf.set("mapred.reducer.class", "testjar.ExternalMapperReducer");
//pass a job.jar already included in the hadoop build
conf.setJar("build/test/mapred/testjar/testjob.jar");
// set the tests jar file
conf.setJarByClass(TestMiniMRClasspath.class);
JobClient.runJob(conf);
StringBuffer result = new StringBuffer();
Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
@ -164,6 +158,7 @@ static String launchExternal(URI uri, String jobTracker, JobConf conf,
return result.toString();
}
@Test
public void testClassPath() throws IOException {
String namenode = null;
MiniDFSCluster dfs = null;
@ -180,13 +175,10 @@ public void testClassPath() throws IOException {
mr = new MiniMRCluster(taskTrackers, namenode, 3);
JobConf jobConf = new JobConf();
String result;
final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
result = launchWordCount(fileSys.getUri(), jobTrackerName, jobConf,
"The quick brown fox\nhas many silly\n" +
"red fox sox\n",
3, 1);
assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
"quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
result = launchWordCount(fileSys.getUri(), jobConf,
"The quick brown fox\nhas many silly\n" + "red fox sox\n", 3, 1);
Assert.assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n"
+ "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
} finally {
if (dfs != null) { dfs.shutdown(); }
@ -195,6 +187,7 @@ public void testClassPath() throws IOException {
}
}
@Test
public void testExternalWritable()
throws IOException {
@ -214,12 +207,10 @@ public void testExternalWritable()
mr = new MiniMRCluster(taskTrackers, namenode, 3);
JobConf jobConf = new JobConf();
String result;
final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
result = launchExternal(fileSys.getUri(), jobTrackerName, jobConf,
"Dennis was here!\nDennis again!",
3, 1);
assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
result = launchExternal(fileSys.getUri(), jobConf,
"Dennis was here!\nDennis again!", 3, 1);
Assert.assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
}
finally {

36
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
View File

@ -17,26 +17,26 @@
*/
package org.apache.hadoop.mapred;
import java.io.*;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.security.*;
import org.junit.Ignore;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
/**
* A JUnit test to test Mini Map-Reduce Cluster with Mini-DFS.
*/
@Ignore
public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
public class TestMiniMRWithDFSWithDistinctUsers {
static final UserGroupInformation DFS_UGI = createUGI("dfs", true);
static final UserGroupInformation ALICE_UGI = createUGI("alice", false);
static final UserGroupInformation BOB_UGI = createUGI("bob", false);
@ -45,7 +45,6 @@ public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
MiniDFSCluster dfs = null;
FileSystem fs = null;
Configuration conf = new Configuration();
String jobTrackerName;
static UserGroupInformation createUGI(String name, boolean issuper) {
String group = issuper? "supergroup": name;
@ -71,9 +70,10 @@ public RunningJob run() throws IOException {
});
rj.waitForCompletion();
assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
Assert.assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
}
@Before
public void setUp() throws Exception {
dfs = new MiniDFSCluster(conf, 4, true, null);
@ -96,29 +96,30 @@ public FileSystem run() throws IOException {
mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
1, null, null, MR_UGI, mrConf);
jobTrackerName = "localhost:" + mr.getJobTrackerPort();
}
@After
public void tearDown() throws Exception {
if (mr != null) { mr.shutdown();}
if (dfs != null) { dfs.shutdown(); }
}
@Test
public void testDistinctUsers() throws Exception {
JobConf job1 = mr.createJobConf();
String input = "The quick brown fox\nhas many silly\n"
+ "red fox sox\n";
Path inDir = new Path("/testing/distinct/input");
Path outDir = new Path("/user/alice/output");
TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1,
input, 2, 1, inDir, outDir);
TestMiniMRClasspath
.configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
runJobAsUser(job1, ALICE_UGI);
JobConf job2 = mr.createJobConf();
Path inDir2 = new Path("/testing/distinct/input2");
Path outDir2 = new Path("/user/bob/output2");
TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2,
input, 2, 1, inDir2, outDir2);
TestMiniMRClasspath.configureWordCount(fs, job2, input, 2, 1, inDir2,
outDir2);
runJobAsUser(job2, BOB_UGI);
}
@ -127,6 +128,7 @@ public void testDistinctUsers() throws Exception {
* task makes lots of spills (more than fit in the spill index cache)
* that it will succeed.
*/
@Test
public void testMultipleSpills() throws Exception {
JobConf job1 = mr.createJobConf();
@ -141,8 +143,8 @@ public void testMultipleSpills() throws Exception {
+ "red fox sox\n";
Path inDir = new Path("/testing/distinct/input");
Path outDir = new Path("/user/alice/output");
TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1,
input, 2, 1, inDir, outDir);
TestMiniMRClasspath
.configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
runJobAsUser(job1, ALICE_UGI);
}
}