From 375584ae0fa49b04e159f41783760208394620df Mon Sep 17 00:00:00 2001 From: Thomas White Date: Wed, 27 Mar 2013 11:47:06 +0000 Subject: [PATCH] MAPREDUCE-5006. Fix failing streaming tests due to MAPREDUCE-4994. Contributed by Sandy Ryza. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461541 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 112 +----------------- .../mapred/LocalClientProtocolProvider.java | 4 +- .../streaming/TestStreamReduceNone.java | 1 + .../streaming/TestStreamXmlRecordReader.java | 2 + 4 files changed, 7 insertions(+), 112 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ed9cf3d5de..4c10ed40d8 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -14,10 +14,6 @@ Trunk (Unreleased) MAPREDUCE-4887. Add RehashPartitioner, to smooth distributions with poor implementations of Object#hashCode(). (Radim Kolar via cutting) - HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows - Azure environments. (See breakdown of tasks below for subtasks and - contributors) - IMPROVEMENTS MAPREDUCE-3787. [Gridmix] Optimize job monitoring and STRESS mode for @@ -75,9 +71,6 @@ Trunk (Unreleased) MAPREDUCE-4735. Make arguments in TestDFSIO case insensitive. (Brandon Li via suresh) - MAPREDUCE-5014. Extend Distcp to accept a custom CopyListing. - (Srikanth Sundarrajan via amareshwari) - BUG FIXES MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant. @@ -161,32 +154,11 @@ Trunk (Unreleased) MAPREDUCE-5012. Typo in javadoc for IdentityMapper class. (Adam Monsen via suresh) - - MAPREDUCE-5078. TestMRAppMaster fails on Windows due to mismatched path - separators. (Chris Nauroth via sseth) - BREAKDOWN OF HADOOP-8562 SUBTASKS + MAPREDUCE-5006. Fix failing streaming tests due to MAPREDUCE-4994. + (Sandy Ryza via tomwhite) - MAPREDUCE-4739. Some MapReduce tests fail to find winutils. - (Chris Nauroth via suresh) - - MAPREDUCE-4780. MapReduce distribution build fails on Windows. - (Chris Nauroth via suresh) - - MAPREDUCE-4790. MapReduce build script would be more readable using abspath. - (Chris Nauroth via suresh) - - MAPREDUCE-4869. Fix TestMapReduceChildJVM. (Chris Nauroth via acmurthy) - - MAPREDUCE-4870. Fix TestMRJobsWithHistoryService. (Chris Nauroth via acmurthy) - - MAPREDUCE-4983. Fixed various platform specific assumptions in various tests, - so that they can pass on Windows too. (Chris Nauroth via vinodkv) - - HADOOP-9372. Fix bad timeout annotations on tests. - (Arpit Agarwal via suresh) - -Release 2.0.5-beta - UNRELEASED +Release 2.0.4-beta - UNRELEASED INCOMPATIBLE CHANGES @@ -197,15 +169,6 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5033. mapred shell script should respect usage flags (--help -help -h). (Andrew Wang via atm) - MAPREDUCE-4892. Modify CombineFileInputFormat to not skew input slits' - allocation on small clusters. (Bikas Saha via vinodkv) - - MAPREDUCE-4990. Construct debug strings conditionally in - ShuffleHandler.Shuffle#sendMapOutput(). (kkambatl via tucu) - - MAPREDUCE-4875. coverage fixing for org.apache.hadoop.mapred - (Aleksey Gorshkov via bobby) - OPTIMIZATIONS BUG FIXES @@ -232,48 +195,6 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5008. Merger progress miscounts with respect to EOF_MARKER. (Sandy Ryza via tomwhite) - MAPREDUCE-4693. History server should include counters for failed tasks. - (Xuan Gong via sseth) - - MAPREDUCE-4896. mapred queue -info spits out ugly exception when queue does - not exist. (sandyr via tucu) - - MAPREDUCE-3685. Fix bugs in MergeManager to ensure compression codec is - appropriately used and that on-disk segments are correctly sorted on - file-size. (Anty Rao and Ravi Prakash via acmurthy) - - MAPREDUCE-4571. TestHsWebServicesJobs fails on jdk7. (tgraves via tucu) - - MAPREDUCE-4716. TestHsWebServicesJobsQuery.testJobsQueryStateInvalid - fails with jdk7. (tgraves via tucu) - - MAPREDUCE-5075. DistCp leaks input file handles since ThrottledInputStream - does not close the wrapped InputStream. (Chris Nauroth via szetszwo) - - MAPREDUCE-3872. Fix an event handling races in ContainerLauncherImpl. - (Robert Kanter via sseth) - - MAPREDUCE-5083. MiniMRCluster should use a random component when creating an - actual cluster (Siddharth Seth via hitesh) - - MAPREDUCE-5062. Fix MR AM to read max-retries from the RM. (Zhijie Shen via - vinodkv) - - MAPREDUCE-5077. Remove mapreduce.util.ResourceCalculatorPlugin and related - code. (Karthik Kambatla via sseth) - -Release 2.0.4-alpha - UNRELEASED - - INCOMPATIBLE CHANGES - - NEW FEATURES - - IMPROVEMENTS - - OPTIMIZATIONS - - BUG FIXES - Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES @@ -804,12 +725,6 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-4989. JSONify DataTables input data for Attempts page (Ravi Prakash via jlowe) - MAPREDUCE-5027. Shuffle does not limit number of outstanding connections - (Robert Parker via jeagles) - - MAPREDUCE-4972. Coverage fixing for org.apache.hadoop.mapreduce.jobhistory - (Aleksey Gorshkov via bobby) - OPTIMIZATIONS MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the @@ -829,27 +744,6 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-5009. Killing the Task Attempt slated for commit does not clear the value from the Task commitAttempt member (Robert Parker via jeagles) - MAPREDUCE-4871. AM uses mapreduce.jobtracker.split.metainfo.maxsize but - mapred-default has mapreduce.job.split.metainfo.maxsize (Jason Lowe via - jeagles) - - MAPREDUCE-4794. DefaultSpeculator generates error messages on normal - shutdown (Jason Lowe via jeagles) - - MAPREDUCE-5043. Fetch failure processing can cause AM event queue to - backup and eventually OOM (Jason Lowe via bobby) - - MAPREDUCE-5023. History Server Web Services missing Job Counters (Ravi - Prakash via tgraves) - - MAPREDUCE-5060. Fetch failures that time out only count against the first - map task (Robert Joseph Evans via jlowe) - - MAPREDUCE-5042. Reducer unable to fetch for a map task that was recovered - (Jason Lowe via bobby) - - MAPREDUCE-5053. java.lang.InternalError from decompression codec cause - reducer to fail (Robert Parker via jeagles) Release 0.23.6 - UNRELEASED diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java index 5b7e3a0b92..3d6aa62adc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java @@ -37,9 +37,7 @@ public ClientProtocol create(Configuration conf) throws IOException { if (!MRConfig.LOCAL_FRAMEWORK_NAME.equals(framework)) { return null; } - if (conf.get("mapreduce.job.maps") == null) { - conf.setInt("mapreduce.job.maps", 1); - } + conf.setInt(JobContext.NUM_MAPS, 1); return new LocalJobRunner(conf); } diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java index 5da9e03a48..766402184c 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java @@ -68,6 +68,7 @@ protected String[] genArgs() { "-reducer", "org.apache.hadoop.mapred.lib.IdentityReducer", "-numReduceTasks", "0", "-jobconf", "mapreduce.task.files.preserve.failedtasks=true", + "-jobconf", "mapreduce.job.maps=1", "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp") }; } diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java index 7a1c6f6106..da0bdae484 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java @@ -54,6 +54,8 @@ protected void createInput() throws IOException protected String[] genArgs() { args.add("-inputreader"); args.add("StreamXmlRecordReader,begin=,end="); + args.add("-jobconf"); + args.add("mapreduce.job.maps=1"); return super.genArgs(); } }