diff --git a/mapreduce/CHANGES.txt b/mapreduce/CHANGES.txt index 988c40eda0..75c454e914 100644 --- a/mapreduce/CHANGES.txt +++ b/mapreduce/CHANGES.txt @@ -186,6 +186,9 @@ Trunk (unreleased changes) BUG FIXES + MAPREDUCE-2603. Disable High-Ram emulation in system tests. + (Vinay Kumar Thota via amarrk) + MAPREDUCE-2539. Fixed NPE in getMapTaskReports in JobClient. (Robert Evans via acmurthy) diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java index 4144bae842..3ade9e34e6 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java @@ -56,6 +56,7 @@ public class TestCompressionEmulationEnableForAllTypesOfJobs final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true", "-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.46", "-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.35", @@ -84,6 +85,7 @@ public class TestCompressionEmulationEnableForAllTypesOfJobs final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java index 6f0dcbff0f..4b7fc3a15a 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java @@ -56,6 +56,7 @@ public class TestCompressionEmulationForCompressInAndUncompressOut final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true" }; @@ -85,6 +86,7 @@ public class TestCompressionEmulationForCompressInAndUncompressOut "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true", "-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.58", "-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.42" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java index 70dc0d1276..383fc83de4 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java @@ -54,6 +54,7 @@ public class TestCompressionEmulationForUncompressInAndCompressOut final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true" }; @@ -82,6 +83,7 @@ public class TestCompressionEmulationForUncompressInAndCompressOut final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_OUTPUT_COMPRESSION_RATIO + "=0.38" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java index d98b259177..a1ae1e9dfa 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java @@ -59,6 +59,7 @@ public class TestEmulationOfHDFSAndLocalFSDCFiles extends final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -85,6 +86,7 @@ public class TestEmulationOfHDFSAndLocalFSDCFiles extends final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java index 00d2e4825a..7f8938f88a 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java @@ -58,6 +58,7 @@ public class TestEmulationOfHDFSDCFileUsesMultipleJobs extends final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -81,6 +82,7 @@ public class TestEmulationOfHDFSDCFileUsesMultipleJobs extends tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java index 3840f1bbea..453e5b9908 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java @@ -60,6 +60,7 @@ public class TestEmulationOfHDFSDCFilesWithDifferentVisibilities final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -81,6 +82,7 @@ public class TestEmulationOfHDFSDCFilesWithDifferentVisibilities "REPLAY", tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java index e50eb6e2e8..eff47f2d64 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java @@ -57,6 +57,7 @@ public class TestEmulationOfLocalFSDCFiles extends GridmixSystemTestCase { final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -83,6 +84,7 @@ public class TestEmulationOfLocalFSDCFiles extends GridmixSystemTestCase { final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java index f1501bf850..ef273b5fd2 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java @@ -93,6 +93,7 @@ public class TestGridMixDataGeneration { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; int exitCode = @@ -123,6 +124,7 @@ public class TestGridMixDataGeneration { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; @@ -154,6 +156,7 @@ public class TestGridMixDataGeneration { long bytesPerFile = 200 * 1024 * 1024; // 200 mb per file of data String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile, + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java index 1ad10d8af5..883feec88f 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java @@ -80,6 +80,7 @@ public class TestGridMixFilePool { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java index adaa0d2363..3fdd16d7f6 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java @@ -59,6 +59,7 @@ public class TestGridmixCompressionEmulationWithCompressInput final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -89,6 +90,7 @@ public class TestGridmixCompressionEmulationWithCompressInput "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true", "-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.68", "-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.35", "-D", GridMixConfig.GRIDMIX_OUTPUT_COMPRESSION_RATIO + "=0.40" diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java index 5289bf3c8a..e6c7e6af46 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java @@ -56,6 +56,7 @@ public class TestGridmixEmulationOfHDFSPrivateDCFile final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -78,6 +79,7 @@ public class TestGridmixEmulationOfHDFSPrivateDCFile "REPLAY", tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java index e12180c72e..0bf07fdf4d 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java @@ -55,6 +55,7 @@ public class TestGridmixEmulationOfHDFSPublicDCFile final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -80,6 +81,7 @@ public class TestGridmixEmulationOfHDFSPublicDCFile tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java index 4dca1a214c..5f464ce39b 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java @@ -58,6 +58,7 @@ public class TestGridmixEmulationOfMultipleHDFSPrivateDCFiles tracePath}; final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -81,6 +82,7 @@ public class TestGridmixEmulationOfMultipleHDFSPrivateDCFiles "STRESS", tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java index 09bbf18122..cca5da83ec 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java @@ -59,6 +59,7 @@ public class TestGridmixEmulationOfMultipleHDFSPublicDCFiles final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -81,6 +82,7 @@ public class TestGridmixEmulationOfMultipleHDFSPublicDCFiles tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java index c48a7461d3..ec11a2b36e 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java @@ -55,6 +55,7 @@ public class TestGridmixWith10minTrace extends GridmixSystemTestCase { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize, "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=false", "-D", GridMixConfig.GRIDMIX_SLEEPJOB_MAPTASK_ONLY + "=true", diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java index ec2a1377bd..9bcb45a3fb 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java @@ -51,6 +51,7 @@ public class TestGridmixWith12minTrace extends GridmixSystemTestCase { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_SLEEP_MAP_MAX_TIME + "=10", "-D", GridMixConfig.GRIDMIX_SLEEP_REDUCE_MAX_TIME + "=5" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java index ed2648448f..c583e6d3a2 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java @@ -49,6 +49,7 @@ public class TestGridmixWith1minTrace extends GridmixSystemTestCase{ String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java index 9628dd2db8..d9fb7c70f7 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java @@ -56,6 +56,7 @@ public class TestGridmixWith2minStreamingJobTrace "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true", "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize, "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath); diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java index 926f795b74..85dedf6675 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java @@ -59,6 +59,7 @@ public class TestGridmixWith3minStreamingJobTrace "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true", "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile, "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java index bed33d0dd3..5f2171fb40 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java @@ -52,6 +52,7 @@ public class TestGridmixWith3minTrace extends GridmixSystemTestCase { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java index 370f120aa9..ef1878c085 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java @@ -56,6 +56,7 @@ public class TestGridmixWith5minStreamingJobTrace "-D", GridMixConfig.GRIDMIX_KEY_FRC + "=0.5f", "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile, "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minTrace.java index 5a141d4c81..c55167e3b4 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minTrace.java @@ -51,6 +51,7 @@ public class TestGridmixWith5minTrace extends GridmixSystemTestCase { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith7minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith7minTrace.java index 0791d68aee..55be37b17d 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith7minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith7minTrace.java @@ -51,6 +51,7 @@ public class TestGridmixWith7minTrace extends GridmixSystemTestCase { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize, "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=false" diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java index 46872b2418..ae71ec5764 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java @@ -316,7 +316,9 @@ public class GridmixJobVerification { Path jhpath = new Path(historyFilePath); fs = jhpath.getFileSystem(conf); fs.copyToLocalFile(jhpath,new Path(tmpJHFolder.toString())); - fs.copyToLocalFile(new Path(historyFilePath + "_conf.xml"), + String historyPath = + historyFilePath.substring(0,historyFilePath.lastIndexOf("_")); + fs.copyToLocalFile(new Path(historyPath + "_conf.xml"), new Path(tmpJHFolder.toString())); JobConf jobConf = new JobConf(); jobConf.addResource(new Path(tmpJHFolder.toString() diff --git a/mapreduce/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case4.json.gz b/mapreduce/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case4.json.gz index c4d4657c3c..229d8d321b 100644 Binary files a/mapreduce/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case4.json.gz and b/mapreduce/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case4.json.gz differ