From 41eacf4914ffc6129f293440a9bcc99723ef3e64 Mon Sep 17 00:00:00 2001 From: Kaiyao Ke <47203510+kaiyaok2@users.noreply.github.com> Date: Fri, 17 May 2024 08:51:47 -0500 Subject: [PATCH] MAPREDUCE-7475. Fix non-idempotent unit tests (#6785) Contributed by Kaiyao Ke --- .../v2/app/webapp/TestAppController.java | 2 ++ .../org/apache/hadoop/mapred/TestMapTask.java | 18 +++++++++++---- .../mapred/TestTaskProgressReporter.java | 6 +++++ .../hadoop/mapred/NotificationTestCase.java | 2 ++ .../mapred/TestOldCombinerGrouping.java | 23 ++++++++++++------- .../mapreduce/TestNewCombinerGrouping.java | 23 ++++++++++++------- 6 files changed, 53 insertions(+), 21 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java index ba5c430121..473681c3e4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java @@ -319,6 +319,8 @@ public void testAttempts() { appController.attempts(); assertEquals(AttemptsPage.class, appController.getClazz()); + + appController.getProperty().remove(AMParams.ATTEMPT_STATE); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java index fef179994f..771a5313ec 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java @@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.util.Progress; import org.junit.After; +import org.junit.Before; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -47,14 +48,21 @@ import static org.mockito.Mockito.mock; public class TestMapTask { - private static File TEST_ROOT_DIR = new File( + private static File testRootDir = new File( System.getProperty("test.build.data", System.getProperty("java.io.tmpdir", "/tmp")), TestMapTask.class.getName()); + @Before + public void setup() throws Exception { + if(!testRootDir.exists()) { + testRootDir.mkdirs(); + } + } + @After public void cleanup() throws Exception { - FileUtil.fullyDelete(TEST_ROOT_DIR); + FileUtil.fullyDelete(testRootDir); } @Rule @@ -66,7 +74,7 @@ public void cleanup() throws Exception { public void testShufflePermissions() throws Exception { JobConf conf = new JobConf(); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); - conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath()); + conf.set(MRConfig.LOCAL_DIR, testRootDir.getAbsolutePath()); MapOutputFile mof = new MROutputFiles(); mof.setConf(conf); TaskAttemptID attemptId = new TaskAttemptID("12345", 1, TaskType.MAP, 1, 1); @@ -98,7 +106,7 @@ public void testShufflePermissions() throws Exception { public void testSpillFilesCountLimitInvalidValue() throws Exception { JobConf conf = new JobConf(); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); - conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath()); + conf.set(MRConfig.LOCAL_DIR, testRootDir.getAbsolutePath()); conf.setInt(MRJobConfig.SPILL_FILES_COUNT_LIMIT, -2); MapOutputFile mof = new MROutputFiles(); mof.setConf(conf); @@ -124,7 +132,7 @@ public void testSpillFilesCountLimitInvalidValue() throws Exception { public void testSpillFilesCountBreach() throws Exception { JobConf conf = new JobConf(); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); - conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath()); + conf.set(MRConfig.LOCAL_DIR, testRootDir.getAbsolutePath()); conf.setInt(MRJobConfig.SPILL_FILES_COUNT_LIMIT, 2); MapOutputFile mof = new MROutputFiles(); mof.setConf(conf); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java index 52875b7aca..93602935c9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java @@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID; import org.apache.hadoop.util.ExitUtil; import org.junit.After; +import org.junit.Before; import org.junit.Assert; import org.junit.Test; @@ -180,6 +181,11 @@ protected void checkTaskLimits() throws TaskLimitException { } } + @Before + public void setup() { + statusUpdateTimes = 0; + } + @After public void cleanup() { FileSystem.clearStatistics(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java index 3372c8f28b..8acd015ab0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java @@ -158,6 +158,8 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { stopHttpServer(); + NotificationServlet.counter = 0; + NotificationServlet.failureCounter = 0; super.tearDown(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java index 046c2d37ee..1f6395dfb7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java @@ -18,11 +18,16 @@ package org.apache.hadoop.mapred; +import org.junit.After; import org.junit.Assert; + +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.Text; +import org.apache.hadoop.test.GenericTestUtils; + import org.junit.Test; import java.io.BufferedReader; @@ -34,12 +39,9 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Set; -import java.util.UUID; public class TestOldCombinerGrouping { - private static String TEST_ROOT_DIR = new File(System.getProperty( - "test.build.data", "build/test/data"), UUID.randomUUID().toString()) - .getAbsolutePath(); + private static File testRootDir = GenericTestUtils.getRandomizedTestDir(); public static class Map implements Mapper { @@ -117,16 +119,21 @@ public int compare(Text o1, Text o2) { } + @After + public void cleanup() { + FileUtil.fullyDelete(testRootDir); + } + @Test public void testCombiner() throws Exception { - if (!new File(TEST_ROOT_DIR).mkdirs()) { - throw new RuntimeException("Could not create test dir: " + TEST_ROOT_DIR); + if (!testRootDir.mkdirs()) { + throw new RuntimeException("Could not create test dir: " + testRootDir); } - File in = new File(TEST_ROOT_DIR, "input"); + File in = new File(testRootDir, "input"); if (!in.mkdirs()) { throw new RuntimeException("Could not create test dir: " + in); } - File out = new File(TEST_ROOT_DIR, "output"); + File out = new File(testRootDir, "output"); PrintWriter pw = new PrintWriter(new FileWriter(new File(in, "data.txt"))); pw.println("A|a,1"); pw.println("A|b,2"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java index c2054f1d4c..df9c6c5e9c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java @@ -18,7 +18,10 @@ package org.apache.hadoop.mapreduce; +import org.junit.After; import org.junit.Assert; + +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.RawComparator; @@ -26,6 +29,8 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; + +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; import java.io.BufferedReader; @@ -36,12 +41,9 @@ import java.io.PrintWriter; import java.util.HashSet; import java.util.Set; -import java.util.UUID; public class TestNewCombinerGrouping { - private static String TEST_ROOT_DIR = new File(System.getProperty( - "test.build.data", "build/test/data"), UUID.randomUUID().toString()) - .getAbsolutePath(); + private static File testRootDir = GenericTestUtils.getRandomizedTestDir(); public static class Map extends Mapper { @@ -103,16 +105,21 @@ public int compare(Text o1, Text o2) { } + @After + public void cleanup() { + FileUtil.fullyDelete(testRootDir); + } + @Test public void testCombiner() throws Exception { - if (!new File(TEST_ROOT_DIR).mkdirs()) { - throw new RuntimeException("Could not create test dir: " + TEST_ROOT_DIR); + if (!testRootDir.mkdirs()) { + throw new RuntimeException("Could not create test dir: " + testRootDir); } - File in = new File(TEST_ROOT_DIR, "input"); + File in = new File(testRootDir, "input"); if (!in.mkdirs()) { throw new RuntimeException("Could not create test dir: " + in); } - File out = new File(TEST_ROOT_DIR, "output"); + File out = new File(testRootDir, "output"); PrintWriter pw = new PrintWriter(new FileWriter(new File(in, "data.txt"))); pw.println("A|a,1"); pw.println("A|b,2");