diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index fe614e1138..8c66a1c74c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -213,7 +213,11 @@ Release 2.0.0 - UNRELEASED MAPREDUCE-4111. Fix tests in org.apache.hadoop.mapred.TestJobName (Devaraj K via bobby) - + + MAPREDUCE-4110. Fix tests in org.apache.hadoop.mapred.TestMiniMRClasspath & + org.apache.hadoop.mapred.TestMiniMRWithDFSWithDistinctUsers (Devaraj K via + bobby) + Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java index 9f8b4a7390..6a81a41552 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java @@ -18,47 +18,43 @@ package org.apache.hadoop.mapred; -import java.io.*; +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; import java.net.URI; import junit.framework.TestCase; + import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.junit.Ignore; +import org.junit.Assert; +import org.junit.Test; /** * A JUnit test to test Mini Map-Reduce Cluster with multiple directories * and check for correct classpath */ -@Ignore -public class TestMiniMRClasspath extends TestCase { +public class TestMiniMRClasspath { - static void configureWordCount(FileSystem fs, - String jobTracker, - JobConf conf, - String input, - int numMaps, - int numReduces, - Path inDir, Path outDir) throws IOException { + static void configureWordCount(FileSystem fs, JobConf conf, String input, + int numMaps, int numReduces, Path inDir, Path outDir) throws IOException { fs.delete(outDir, true); if (!fs.mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir.toString()); } - { - DataOutputStream file = fs.create(new Path(inDir, "part-0")); - file.writeBytes(input); - file.close(); - } + DataOutputStream file = fs.create(new Path(inDir, "part-0")); + file.writeBytes(input); + file.close(); FileSystem.setDefaultUri(conf, fs.getUri()); - conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME); - conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker); + conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME); conf.setJobName("wordcount"); conf.setInputFormat(TextInputFormat.class); @@ -74,18 +70,17 @@ static void configureWordCount(FileSystem fs, FileOutputFormat.setOutputPath(conf, outDir); conf.setNumMapTasks(numMaps); conf.setNumReduceTasks(numReduces); - //pass a job.jar already included in the hadoop build - conf.setJar("build/test/mapred/testjar/testjob.jar"); + //set the tests jar file + conf.setJarByClass(TestMiniMRClasspath.class); } - static String launchWordCount(URI fileSys, String jobTracker, JobConf conf, - String input, int numMaps, int numReduces) + static String launchWordCount(URI fileSys, JobConf conf, String input, + int numMaps, int numReduces) throws IOException { final Path inDir = new Path("/testing/wc/input"); final Path outDir = new Path("/testing/wc/output"); FileSystem fs = FileSystem.get(fileSys, conf); - configureWordCount(fs, jobTracker, conf, input, numMaps, numReduces, inDir, - outDir); + configureWordCount(fs, conf, input, numMaps, numReduces, inDir, outDir); JobClient.runJob(conf); StringBuffer result = new StringBuffer(); { @@ -107,8 +102,8 @@ static String launchWordCount(URI fileSys, String jobTracker, JobConf conf, return result.toString(); } - static String launchExternal(URI uri, String jobTracker, JobConf conf, - String input, int numMaps, int numReduces) + static String launchExternal(URI uri, JobConf conf, String input, + int numMaps, int numReduces) throws IOException { final Path inDir = new Path("/testing/ext/input"); @@ -124,8 +119,7 @@ static String launchExternal(URI uri, String jobTracker, JobConf conf, file.close(); } FileSystem.setDefaultUri(conf, uri); - conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME); - conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker); + conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME); conf.setJobName("wordcount"); conf.setInputFormat(TextInputFormat.class); @@ -142,8 +136,8 @@ static String launchExternal(URI uri, String jobTracker, JobConf conf, conf.set("mapred.mapper.class", "testjar.ExternalMapperReducer"); conf.set("mapred.reducer.class", "testjar.ExternalMapperReducer"); - //pass a job.jar already included in the hadoop build - conf.setJar("build/test/mapred/testjar/testjob.jar"); + // set the tests jar file + conf.setJarByClass(TestMiniMRClasspath.class); JobClient.runJob(conf); StringBuffer result = new StringBuffer(); Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir, @@ -164,6 +158,7 @@ static String launchExternal(URI uri, String jobTracker, JobConf conf, return result.toString(); } + @Test public void testClassPath() throws IOException { String namenode = null; MiniDFSCluster dfs = null; @@ -180,13 +175,10 @@ public void testClassPath() throws IOException { mr = new MiniMRCluster(taskTrackers, namenode, 3); JobConf jobConf = new JobConf(); String result; - final String jobTrackerName = "localhost:" + mr.getJobTrackerPort(); - result = launchWordCount(fileSys.getUri(), jobTrackerName, jobConf, - "The quick brown fox\nhas many silly\n" + - "red fox sox\n", - 3, 1); - assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" + - "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result); + result = launchWordCount(fileSys.getUri(), jobConf, + "The quick brown fox\nhas many silly\n" + "red fox sox\n", 3, 1); + Assert.assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" + + "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result); } finally { if (dfs != null) { dfs.shutdown(); } @@ -195,6 +187,7 @@ public void testClassPath() throws IOException { } } + @Test public void testExternalWritable() throws IOException { @@ -214,12 +207,10 @@ public void testExternalWritable() mr = new MiniMRCluster(taskTrackers, namenode, 3); JobConf jobConf = new JobConf(); String result; - final String jobTrackerName = "localhost:" + mr.getJobTrackerPort(); - result = launchExternal(fileSys.getUri(), jobTrackerName, jobConf, - "Dennis was here!\nDennis again!", - 3, 1); - assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result); + result = launchExternal(fileSys.getUri(), jobConf, + "Dennis was here!\nDennis again!", 3, 1); + Assert.assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result); } finally { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java index 7ebf8c7e77..041f2dbfbf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java @@ -17,26 +17,26 @@ */ package org.apache.hadoop.mapred; -import java.io.*; +import java.io.IOException; import java.security.PrivilegedExceptionAction; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.security.*; -import org.junit.Ignore; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; /** * A JUnit test to test Mini Map-Reduce Cluster with Mini-DFS. */ -@Ignore -public class TestMiniMRWithDFSWithDistinctUsers extends TestCase { +public class TestMiniMRWithDFSWithDistinctUsers { static final UserGroupInformation DFS_UGI = createUGI("dfs", true); static final UserGroupInformation ALICE_UGI = createUGI("alice", false); static final UserGroupInformation BOB_UGI = createUGI("bob", false); @@ -45,7 +45,6 @@ public class TestMiniMRWithDFSWithDistinctUsers extends TestCase { MiniDFSCluster dfs = null; FileSystem fs = null; Configuration conf = new Configuration(); - String jobTrackerName; static UserGroupInformation createUGI(String name, boolean issuper) { String group = issuper? "supergroup": name; @@ -71,9 +70,10 @@ public RunningJob run() throws IOException { }); rj.waitForCompletion(); - assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState())); + Assert.assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState())); } + @Before public void setUp() throws Exception { dfs = new MiniDFSCluster(conf, 4, true, null); @@ -96,29 +96,30 @@ public FileSystem run() throws IOException { mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(), 1, null, null, MR_UGI, mrConf); - jobTrackerName = "localhost:" + mr.getJobTrackerPort(); } + @After public void tearDown() throws Exception { if (mr != null) { mr.shutdown();} if (dfs != null) { dfs.shutdown(); } } + @Test public void testDistinctUsers() throws Exception { JobConf job1 = mr.createJobConf(); String input = "The quick brown fox\nhas many silly\n" + "red fox sox\n"; Path inDir = new Path("/testing/distinct/input"); Path outDir = new Path("/user/alice/output"); - TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, - input, 2, 1, inDir, outDir); + TestMiniMRClasspath + .configureWordCount(fs, job1, input, 2, 1, inDir, outDir); runJobAsUser(job1, ALICE_UGI); JobConf job2 = mr.createJobConf(); Path inDir2 = new Path("/testing/distinct/input2"); Path outDir2 = new Path("/user/bob/output2"); - TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2, - input, 2, 1, inDir2, outDir2); + TestMiniMRClasspath.configureWordCount(fs, job2, input, 2, 1, inDir2, + outDir2); runJobAsUser(job2, BOB_UGI); } @@ -127,6 +128,7 @@ public void testDistinctUsers() throws Exception { * task makes lots of spills (more than fit in the spill index cache) * that it will succeed. */ + @Test public void testMultipleSpills() throws Exception { JobConf job1 = mr.createJobConf(); @@ -141,8 +143,8 @@ public void testMultipleSpills() throws Exception { + "red fox sox\n"; Path inDir = new Path("/testing/distinct/input"); Path outDir = new Path("/user/alice/output"); - TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, - input, 2, 1, inDir, outDir); + TestMiniMRClasspath + .configureWordCount(fs, job1, input, 2, 1, inDir, outDir); runJobAsUser(job1, ALICE_UGI); } }