MAPREDUCE-4110. Fix tests in TestMiniMRClasspath & TestMiniMRWithDFSWithDistinctUsers (Devaraj K via bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1310546 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -214,6 +214,10 @@ Release 2.0.0 - UNRELEASED
     MAPREDUCE-4111. Fix tests in org.apache.hadoop.mapred.TestJobName (Devaraj
     K via bobby)
 
+    MAPREDUCE-4110. Fix tests in org.apache.hadoop.mapred.TestMiniMRClasspath &
+    org.apache.hadoop.mapred.TestMiniMRWithDFSWithDistinctUsers (Devaraj K via
+    bobby)
+
 Release 0.23.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java

@@ -18,47 +18,43 @@
 
 package org.apache.hadoop.mapred;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.net.URI;
 
 import junit.framework.TestCase;
+
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.junit.Ignore;
+import org.junit.Assert;
+import org.junit.Test;
 
 /**
  * A JUnit test to test Mini Map-Reduce Cluster with multiple directories
  * and check for correct classpath
  */
-@Ignore
+public class TestMiniMRClasspath {
-public class TestMiniMRClasspath extends TestCase {
   
   
-  static void configureWordCount(FileSystem fs,
+  static void configureWordCount(FileSystem fs, JobConf conf, String input,
-                                String jobTracker,
+      int numMaps, int numReduces, Path inDir, Path outDir) throws IOException {
-                                JobConf conf,
-                                String input,
-                                int numMaps,
-                                int numReduces,
-                                Path inDir, Path outDir) throws IOException {
     fs.delete(outDir, true);
     if (!fs.mkdirs(inDir)) {
       throw new IOException("Mkdirs failed to create " + inDir.toString());
     }
-    {
+    DataOutputStream file = fs.create(new Path(inDir, "part-0"));
-      DataOutputStream file = fs.create(new Path(inDir, "part-0"));
+    file.writeBytes(input);
-      file.writeBytes(input);
+    file.close();
-      file.close();
-    }
     FileSystem.setDefaultUri(conf, fs.getUri());
-    conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
+    conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
-    conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
     conf.setJobName("wordcount");
     conf.setInputFormat(TextInputFormat.class);
     
@@ -74,18 +70,17 @@ static void configureWordCount(FileSystem fs,
     FileOutputFormat.setOutputPath(conf, outDir);
     conf.setNumMapTasks(numMaps);
     conf.setNumReduceTasks(numReduces);
-    //pass a job.jar already included in the hadoop build
+    //set the tests jar file
-    conf.setJar("build/test/mapred/testjar/testjob.jar");
+    conf.setJarByClass(TestMiniMRClasspath.class);
   }
   
-  static String launchWordCount(URI fileSys, String jobTracker, JobConf conf,
+  static String launchWordCount(URI fileSys, JobConf conf, String input,
-                                String input, int numMaps, int numReduces) 
+                                int numMaps, int numReduces) 
   throws IOException {
     final Path inDir = new Path("/testing/wc/input");
     final Path outDir = new Path("/testing/wc/output");
     FileSystem fs = FileSystem.get(fileSys, conf);
-    configureWordCount(fs, jobTracker, conf, input, numMaps, numReduces, inDir, 
+    configureWordCount(fs, conf, input, numMaps, numReduces, inDir, outDir);
-                       outDir);
     JobClient.runJob(conf);
     StringBuffer result = new StringBuffer();
     {
@@ -107,8 +102,8 @@ static String launchWordCount(URI fileSys, String jobTracker, JobConf conf,
     return result.toString();
   }
 
-  static String launchExternal(URI uri, String jobTracker, JobConf conf,
+  static String launchExternal(URI uri, JobConf conf, String input,
-                               String input, int numMaps, int numReduces)
+                               int numMaps, int numReduces)
     throws IOException {
 
     final Path inDir = new Path("/testing/ext/input");
@@ -124,8 +119,7 @@ static String launchExternal(URI uri, String jobTracker, JobConf conf,
       file.close();
     }
     FileSystem.setDefaultUri(conf, uri);
-    conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
+    conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
-    conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
     conf.setJobName("wordcount");
     conf.setInputFormat(TextInputFormat.class);
 
@@ -142,8 +136,8 @@ static String launchExternal(URI uri, String jobTracker, JobConf conf,
     conf.set("mapred.mapper.class", "testjar.ExternalMapperReducer"); 
     conf.set("mapred.reducer.class", "testjar.ExternalMapperReducer");
 
-    //pass a job.jar already included in the hadoop build
+    // set the tests jar file
-    conf.setJar("build/test/mapred/testjar/testjob.jar");
+    conf.setJarByClass(TestMiniMRClasspath.class);
     JobClient.runJob(conf);
     StringBuffer result = new StringBuffer();
     Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
@@ -164,6 +158,7 @@ static String launchExternal(URI uri, String jobTracker, JobConf conf,
     return result.toString();
   }
    
+  @Test
   public void testClassPath() throws IOException {
     String namenode = null;
     MiniDFSCluster dfs = null;
@@ -180,13 +175,10 @@ public void testClassPath() throws IOException {
       mr = new MiniMRCluster(taskTrackers, namenode, 3);
       JobConf jobConf = new JobConf();
       String result;
-      final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
+      result = launchWordCount(fileSys.getUri(), jobConf,
-      result = launchWordCount(fileSys.getUri(), jobTrackerName, jobConf, 
+          "The quick brown fox\nhas many silly\n" + "red fox sox\n", 3, 1);
-                               "The quick brown fox\nhas many silly\n" + 
+      Assert.assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n"
-                               "red fox sox\n",
+          + "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
-                               3, 1);
-      assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
-                   "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
           
     } finally {
       if (dfs != null) { dfs.shutdown(); }
@@ -195,6 +187,7 @@ public void testClassPath() throws IOException {
     }
   }
   
+  @Test
   public void testExternalWritable()
     throws IOException {
  
@@ -214,12 +207,10 @@ public void testExternalWritable()
       mr = new MiniMRCluster(taskTrackers, namenode, 3);      
       JobConf jobConf = new JobConf();
       String result;
-      final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
       
-      result = launchExternal(fileSys.getUri(), jobTrackerName, jobConf, 
+      result = launchExternal(fileSys.getUri(), jobConf,
-                              "Dennis was here!\nDennis again!",
+          "Dennis was here!\nDennis again!", 3, 1);
-                              3, 1);
+      Assert.assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
-      assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
       
     } 
     finally {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java

@@ -17,26 +17,26 @@
  */
 package org.apache.hadoop.mapred;
 
-import java.io.*;
+import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 
-import junit.framework.TestCase;
-
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.security.*;
+import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.Ignore;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
 
 /**
  * A JUnit test to test Mini Map-Reduce Cluster with Mini-DFS.
  */
-@Ignore
+public class TestMiniMRWithDFSWithDistinctUsers {
-public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
   static final UserGroupInformation DFS_UGI = createUGI("dfs", true); 
   static final UserGroupInformation ALICE_UGI = createUGI("alice", false); 
   static final UserGroupInformation BOB_UGI = createUGI("bob", false); 
@@ -45,7 +45,6 @@ public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
   MiniDFSCluster dfs = null;
   FileSystem fs = null;
   Configuration conf = new Configuration();
-  String jobTrackerName;
 
   static UserGroupInformation createUGI(String name, boolean issuper) {
     String group = issuper? "supergroup": name;
@@ -71,9 +70,10 @@ public RunningJob run() throws IOException {
       });
 
     rj.waitForCompletion();
-    assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
+    Assert.assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
   }
 
+  @Before
   public void setUp() throws Exception {
     dfs = new MiniDFSCluster(conf, 4, true, null);
 
@@ -96,29 +96,30 @@ public FileSystem run() throws IOException {
 
     mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
                            1, null, null, MR_UGI, mrConf);
-    jobTrackerName = "localhost:" + mr.getJobTrackerPort();
   }
 
+  @After
   public void tearDown() throws Exception {
     if (mr != null) { mr.shutdown();}
     if (dfs != null) { dfs.shutdown(); }
   }
   
+  @Test
   public void testDistinctUsers() throws Exception {
     JobConf job1 = mr.createJobConf();
     String input = "The quick brown fox\nhas many silly\n" 
       + "red fox sox\n";
     Path inDir = new Path("/testing/distinct/input");
     Path outDir = new Path("/user/alice/output");
-    TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, 
+    TestMiniMRClasspath
-                                           input, 2, 1, inDir, outDir);
+        .configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
     runJobAsUser(job1, ALICE_UGI);
 
     JobConf job2 = mr.createJobConf();
     Path inDir2 = new Path("/testing/distinct/input2");
     Path outDir2 = new Path("/user/bob/output2");
-    TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2, 
+    TestMiniMRClasspath.configureWordCount(fs, job2, input, 2, 1, inDir2,
-                                           input, 2, 1, inDir2, outDir2);
+        outDir2);
     runJobAsUser(job2, BOB_UGI);
   }
 
@@ -127,6 +128,7 @@ public void testDistinctUsers() throws Exception {
    * task makes lots of spills (more than fit in the spill index cache)
    * that it will succeed.
    */
+  @Test
   public void testMultipleSpills() throws Exception {
     JobConf job1 = mr.createJobConf();
 
@@ -141,8 +143,8 @@ public void testMultipleSpills() throws Exception {
       + "red fox sox\n";
     Path inDir = new Path("/testing/distinct/input");
     Path outDir = new Path("/user/alice/output");
-    TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, 
+    TestMiniMRClasspath
-                                           input, 2, 1, inDir, outDir);
+        .configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
     runJobAsUser(job1, ALICE_UGI);
   }
 }