diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 397f94a4d1..4166e6aa95 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -105,6 +105,9 @@ Trunk (Unreleased) MAPREDUCE-6260. Convert site documentation to markdown (Masatake Iwasaki via aw) + MAPREDUCE-6057. Remove obsolete entries from mapred-default.xml + (Ray Chiang via aw) + BUG FIXES MAPREDUCE-6191. Improve clearing stale state of Java serialization diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java index c4957b7551..a5232911de 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java @@ -978,8 +978,9 @@ public void init(MapOutputCollector.Context context throw new IOException( "Invalid \"" + JobContext.IO_SORT_MB + "\": " + sortmb); } - sorter = ReflectionUtils.newInstance(job.getClass("map.sort.class", - QuickSort.class, IndexedSorter.class), job); + sorter = ReflectionUtils.newInstance(job.getClass( + MRJobConfig.MAP_SORT_CLASS, QuickSort.class, + IndexedSorter.class), job); // buffers and accounting int maxMemUsage = sortmb << 20; maxMemUsage -= maxMemUsage % METASIZE; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index e64b9b6ea7..0c2a2e9c82 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -28,6 +28,9 @@ @InterfaceStability.Evolving public interface MRJobConfig { + // Used by MapTask + public static final String MAP_SORT_CLASS = "map.sort.class"; + // Put all of the attribute names in here so that Job and JobContext are // consistent. public static final String INPUT_FORMAT_CLASS_ATTR = "mapreduce.job.inputformat.class"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 820c1ac8d8..ff4511fbfd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -57,13 +57,6 @@ set to less than .5 - - mapreduce.local.clientfactory.class.name - org.apache.hadoop.mapred.LocalClientFactory - This the client factory that is responsible for - creating local job runner client - - mapreduce.job.maps 2 @@ -883,30 +876,6 @@ - - mapreduce.map.skip.proc.count.autoincr - true - The flag which if set to true, - SkipBadRecords.COUNTER_MAP_PROCESSED_RECORDS is incremented - by MapRunner after invoking the map function. This value must be set to - false for applications which process the records asynchronously - or buffer the input records. For example streaming. - In such cases applications should increment this counter on their own. - - - - - mapreduce.reduce.skip.proc.count.autoincr - true - The flag which if set to true, - SkipBadRecords.COUNTER_REDUCE_PROCESSED_GROUPS is incremented - by framework after invoking the reduce function. This value must be set to - false for applications which process the records asynchronously - or buffer the input records. For example streaming. - In such cases applications should increment this counter on their own. - - - mapreduce.job.skip.outdir diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml index 6d1c175f01..c469f786c2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml @@ -93,7 +93,6 @@ mapreduce.reduce.input.buffer.percent0.0 mapreduce.map.output.compress.codecorg.apache.hadoop.io.compress.DefaultCodec yarn.resourcemanager.delegation-token.keepalive-time-ms300000 -mapreduce.map.skip.proc.count.autoincrtrue dfs.datanode.directoryscan.threads1 mapreduce.jobtracker.addresslocal mapreduce.cluster.local.dir${hadoop.tmp.dir}/mapred/local @@ -197,7 +196,6 @@ dfs.block.access.key.update.interval600 mapreduce.jobhistory.move.interval-ms30000 dfs.datanode.dns.interfacedefault -mapreduce.reduce.skip.proc.count.autoincrtrue dfs.namenode.backup.http-address0.0.0.0:50105 yarn.nodemanager.container-monitor.interval-ms3000 mapred.reducer.new-apitrue @@ -290,7 +288,6 @@ yarn.app.mapreduce.am.command-opts-Xmx500m mapreduce.admin.user.envLD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native dfs.namenode.checkpoint.edits.dir${dfs.namenode.checkpoint.dir} -mapreduce.local.clientfactory.class.nameorg.apache.hadoop.mapred.LocalClientFactory hadoop.common.configuration.version0.23.0 mapreduce.tasktracker.dns.interfacedefault io.serializationsorg.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization @@ -380,7 +377,6 @@ fs.AbstractFileSystem.viewfs.implorg.apache.hadoop.fs.viewfs.ViewFs yarn.resourcemanager.resource-tracker.client.thread-count50 mapreduce.tasktracker.dns.nameserverdefault -mapreduce.clientfactory.class.nameorg.apache.hadoop.mapred.YarnClientFactory mapreduce.map.output.compressfalse mapreduce.job.counters.limit120 dfs.datanode.ipc.address0.0.0.0:50020 diff --git a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json index f54c26fa4f..4be3e08a61 100644 --- a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json +++ b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json @@ -4645,7 +4645,6 @@ "dfs.ha.log-roll.period" : "120", "mapreduce.reduce.input.buffer.percent" : "0.0", "mapreduce.map.output.compress.codec" : "org.apache.hadoop.io.compress.SnappyCodec", - "mapreduce.map.skip.proc.count.autoincr" : "true", "dfs.client.failover.sleep.base.millis" : "500", "dfs.datanode.directoryscan.threads" : "1", "mapreduce.jobtracker.address" : "neededForHive:999999", @@ -4765,7 +4764,6 @@ "dfs.namenode.backup.address" : "0.0.0.0:50100", "hadoop.util.hash.type" : "murmur", "dfs.block.access.key.update.interval" : "600", - "mapreduce.reduce.skip.proc.count.autoincr" : "true", "dfs.datanode.dns.interface" : "default", "dfs.datanode.use.datanode.hostname" : "false", "mapreduce.job.output.key.class" : "org.apache.hadoop.io.Text", @@ -4875,7 +4873,6 @@ "mapreduce.reduce.shuffle.read.timeout" : "180000", "mapreduce.admin.user.env" : "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native", "yarn.app.mapreduce.am.command-opts" : " -Xmx1238932873", - "mapreduce.local.clientfactory.class.name" : "org.apache.hadoop.mapred.LocalClientFactory", "dfs.namenode.checkpoint.edits.dir" : "${dfs.namenode.checkpoint.dir}", "fs.permissions.umask-mode" : "022", "dfs.client.domain.socket.data.traffic" : "false", @@ -9754,7 +9751,6 @@ "dfs.ha.log-roll.period" : "120", "mapreduce.reduce.input.buffer.percent" : "0.0", "mapreduce.map.output.compress.codec" : "org.apache.hadoop.io.compress.SnappyCodec", - "mapreduce.map.skip.proc.count.autoincr" : "true", "dfs.client.failover.sleep.base.millis" : "500", "dfs.datanode.directoryscan.threads" : "1", "mapreduce.jobtracker.address" : "neededForHive:999999", @@ -9874,7 +9870,6 @@ "dfs.namenode.backup.address" : "0.0.0.0:50100", "hadoop.util.hash.type" : "murmur", "dfs.block.access.key.update.interval" : "600", - "mapreduce.reduce.skip.proc.count.autoincr" : "true", "dfs.datanode.dns.interface" : "default", "dfs.datanode.use.datanode.hostname" : "false", "mapreduce.job.output.key.class" : "org.apache.hadoop.io.Text", @@ -9984,7 +9979,6 @@ "mapreduce.reduce.shuffle.read.timeout" : "180000", "mapreduce.admin.user.env" : "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native", "yarn.app.mapreduce.am.command-opts" : " -Xmx1238932873", - "mapreduce.local.clientfactory.class.name" : "org.apache.hadoop.mapred.LocalClientFactory", "dfs.namenode.checkpoint.edits.dir" : "${dfs.namenode.checkpoint.dir}", "fs.permissions.umask-mode" : "022", "dfs.client.domain.socket.data.traffic" : "false",