diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md index 94ed68e36a..8de1be21f1 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md @@ -60,7 +60,6 @@ The following table lists the configuration property names that are deprecated i | fs.checkpoint.period | dfs.namenode.checkpoint.period | | fs.default.name | fs.defaultFS | | hadoop.configured.node.mapping | net.topology.configured.node.mapping | -| hadoop.job.history.location | mapreduce.jobtracker.jobhistory.location | | hadoop.native.lib | io.native.lib.available | | hadoop.net.static.resolutions | mapreduce.tasktracker.net.static.resolutions | | hadoop.pipes.command-file.keep | mapreduce.pipes.commandfile.preserve | @@ -102,7 +101,6 @@ The following table lists the configuration property names that are deprecated i | mapred.cache.localArchives | mapreduce.job.cache.local.archives | | mapred.cache.localFiles | mapreduce.job.cache.local.files | | mapred.child.tmp | mapreduce.task.tmp.dir | -| mapred.cluster.average.blacklist.threshold | mapreduce.jobtracker.blacklist.average.threshold | | mapred.cluster.map.memory.mb | mapreduce.cluster.mapmemory.mb | | mapred.cluster.max.map.memory.mb | mapreduce.jobtracker.maxmapmemory.mb | | mapred.cluster.max.reduce.memory.mb | mapreduce.jobtracker.maxreducememory.mb | @@ -115,9 +113,6 @@ The following table lists the configuration property names that are deprecated i | mapred.healthChecker.script.args | mapreduce.tasktracker.healthchecker.script.args | | mapred.healthChecker.script.path | mapreduce.tasktracker.healthchecker.script.path | | mapred.healthChecker.script.timeout | mapreduce.tasktracker.healthchecker.script.timeout | -| mapred.heartbeats.in.second | mapreduce.jobtracker.heartbeats.in.second | -| mapred.hosts.exclude | mapreduce.jobtracker.hosts.exclude.filename | -| mapred.hosts | mapreduce.jobtracker.hosts.filename | | mapred.inmem.merge.threshold | mapreduce.reduce.merge.inmem.threshold | | mapred.input.dir.formats | mapreduce.input.multipleinputs.dir.formats | | mapred.input.dir.mappers | mapreduce.input.multipleinputs.dir.mappers | @@ -127,7 +122,6 @@ The following table lists the configuration property names that are deprecated i | mapred.job.classpath.archives | mapreduce.job.classpath.archives | | mapred.job.classpath.files | mapreduce.job.classpath.files | | mapred.job.id | mapreduce.job.id | -| mapred.jobinit.threads | mapreduce.jobtracker.jobinit.threads | | mapred.job.map.memory.mb | mapreduce.map.memory.mb | | mapred.job.name | mapreduce.job.name | | mapred.job.priority | mapreduce.job.priority | @@ -139,23 +133,9 @@ The following table lists the configuration property names that are deprecated i | mapred.job.reuse.jvm.num.tasks | mapreduce.job.jvm.numtasks | | mapred.job.shuffle.input.buffer.percent | mapreduce.reduce.shuffle.input.buffer.percent | | mapred.job.shuffle.merge.percent | mapreduce.reduce.shuffle.merge.percent | -| mapred.job.tracker.handler.count | mapreduce.jobtracker.handler.count | -| mapred.job.tracker.history.completed.location | mapreduce.jobtracker.jobhistory.completed.location | -| mapred.job.tracker.http.address | mapreduce.jobtracker.http.address | -| mapred.jobtracker.instrumentation | mapreduce.jobtracker.instrumentation | -| mapred.jobtracker.job.history.block.size | mapreduce.jobtracker.jobhistory.block.size | -| mapred.job.tracker.jobhistory.lru.cache.size | mapreduce.jobtracker.jobhistory.lru.cache.size | | mapred.job.tracker | mapreduce.jobtracker.address | -| mapred.jobtracker.maxtasks.per.job | mapreduce.jobtracker.maxtasks.perjob | | mapred.job.tracker.persist.jobstatus.active | mapreduce.jobtracker.persist.jobstatus.active | -| mapred.job.tracker.persist.jobstatus.dir | mapreduce.jobtracker.persist.jobstatus.dir | -| mapred.job.tracker.persist.jobstatus.hours | mapreduce.jobtracker.persist.jobstatus.hours | -| mapred.jobtracker.restart.recover | mapreduce.jobtracker.restart.recover | -| mapred.job.tracker.retiredjobs.cache.size | mapreduce.jobtracker.retiredjobs.cache.size | | mapred.job.tracker.retire.jobs | mapreduce.jobtracker.retirejobs | -| mapred.jobtracker.taskalloc.capacitypad | mapreduce.jobtracker.taskscheduler.taskalloc.capacitypad | -| mapred.jobtracker.taskScheduler | mapreduce.jobtracker.taskscheduler | -| mapred.jobtracker.taskScheduler.maxRunningTasksPerJob | mapreduce.jobtracker.taskscheduler.maxrunningtasks.perjob | | mapred.join.expr | mapreduce.join.expr | | mapred.join.keycomparator | mapreduce.join.keycomparator | | mapred.lazy.output.format | mapreduce.output.lazyoutputformat.outputformat | @@ -179,7 +159,6 @@ The following table lists the configuration property names that are deprecated i | mapred.max.map.failures.percent | mapreduce.map.failures.maxpercent | | mapred.max.reduce.failures.percent | mapreduce.reduce.failures.maxpercent | | mapred.max.split.size | mapreduce.input.fileinputformat.split.maxsize | -| mapred.max.tracker.blacklists | mapreduce.jobtracker.tasktracker.maxblacklists | | mapred.max.tracker.failures | mapreduce.job.maxtaskfailures.per.tracker | | mapred.merge.recordsBeforeProgress | mapreduce.task.merge.progress.records | | mapred.min.split.size | mapreduce.input.fileinputformat.split.minsize | @@ -232,7 +211,6 @@ The following table lists the configuration property names that are deprecated i | mapred.tasktracker.dns.interface | mapreduce.tasktracker.dns.interface | | mapred.tasktracker.dns.nameserver | mapreduce.tasktracker.dns.nameserver | | mapred.tasktracker.events.batchsize | mapreduce.tasktracker.events.batchsize | -| mapred.tasktracker.expiry.interval | mapreduce.jobtracker.expire.trackers.interval | | mapred.task.tracker.http.address | mapreduce.tasktracker.http.address | | mapred.tasktracker.indexcache.mb | mapreduce.tasktracker.indexcache.mb | | mapred.tasktracker.instrumentation | mapreduce.tasktracker.instrumentation | @@ -251,7 +229,6 @@ The following table lists the configuration property names that are deprecated i | mapreduce.combine.class | mapreduce.job.combine.class | | mapreduce.inputformat.class | mapreduce.job.inputformat.class | | mapreduce.job.counters.limit | mapreduce.job.counters.max | -| mapreduce.jobtracker.permissions.supergroup | mapreduce.cluster.permissions.supergroup | | mapreduce.map.class | mapreduce.job.map.class | | mapreduce.outputformat.class | mapreduce.job.outputformat.class | | mapreduce.partitioner.class | mapreduce.job.partitioner.class | @@ -277,7 +254,6 @@ The following table lists the configuration property names that are deprecated i | topology.script.file.name | net.topology.script.file.name | | topology.script.number.args | net.topology.script.number.args | | user.name | mapreduce.job.user.name | -| webinterface.private.actions | mapreduce.jobtracker.webinterface.trusted | | yarn.app.mapreduce.yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | | yarn.client.app-submission.poll-interval | yarn.client.application-client-protocol.poll-timeout-ms | diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java index ef05d2a128..45c05938b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java @@ -31,92 +31,19 @@ public interface JTConfig extends MRConfig { // JobTracker configuration parameters public static final String JT_IPC_ADDRESS = "mapreduce.jobtracker.address"; - public static final String JT_HTTP_ADDRESS = - "mapreduce.jobtracker.http.address"; - public static final String JT_IPC_HANDLER_COUNT = - "mapreduce.jobtracker.handler.count"; - public static final String JT_RESTART_ENABLED = - "mapreduce.jobtracker.restart.recover"; - public static final String JT_TASK_SCHEDULER = - "mapreduce.jobtracker.taskscheduler"; - public static final String JT_INSTRUMENTATION = - "mapreduce.jobtracker.instrumentation"; - public static final String JT_TASKS_PER_JOB = - "mapreduce.jobtracker.maxtasks.perjob"; - public static final String JT_HEARTBEATS_IN_SECOND = - "mapreduce.jobtracker.heartbeats.in.second"; - public static final String JT_HEARTBEATS_SCALING_FACTOR = - "mapreduce.jobtracker.heartbeats.scaling.factor"; - public static final String JT_HEARTBEAT_INTERVAL_MIN = - "mapreduce.jobtracker.heartbeat.interval.min"; - public static final int JT_HEARTBEAT_INTERVAL_MIN_DEFAULT = 300; - public static final String JT_PERSIST_JOBSTATUS = + public static final String JT_PERSIST_JOBSTATUS = "mapreduce.jobtracker.persist.jobstatus.active"; - public static final String JT_PERSIST_JOBSTATUS_HOURS = - "mapreduce.jobtracker.persist.jobstatus.hours"; - public static final String JT_PERSIST_JOBSTATUS_DIR = - "mapreduce.jobtracker.persist.jobstatus.dir"; - /** - * @deprecated Use MR_SUPERGROUP instead - */ - @Deprecated - public static final String JT_SUPERGROUP = - "mapreduce.jobtracker.permissions.supergroup"; - public static final String JT_RETIREJOBS = + public static final String JT_RETIREJOBS = "mapreduce.jobtracker.retirejobs"; - public static final String JT_RETIREJOB_CACHE_SIZE = - "mapreduce.jobtracker.retiredjobs.cache.size"; - public static final String JT_TASKCACHE_LEVELS = + public static final String JT_TASKCACHE_LEVELS = "mapreduce.jobtracker.taskcache.levels"; - public static final String JT_TASK_ALLOC_PAD_FRACTION = - "mapreduce.jobtracker.taskscheduler.taskalloc.capacitypad"; - public static final String JT_JOBINIT_THREADS = - "mapreduce.jobtracker.jobinit.threads"; - public static final String JT_TRACKER_EXPIRY_INTERVAL = - "mapreduce.jobtracker.expire.trackers.interval"; - public static final String JT_RUNNINGTASKS_PER_JOB = - "mapreduce.jobtracker.taskscheduler.maxrunningtasks.perjob"; - public static final String JT_HOSTS_FILENAME = - "mapreduce.jobtracker.hosts.filename"; - public static final String JT_HOSTS_EXCLUDE_FILENAME = - "mapreduce.jobtracker.hosts.exclude.filename"; - public static final String JT_JOBHISTORY_CACHE_SIZE = - "mapreduce.jobtracker.jobhistory.lru.cache.size"; - public static final String JT_JOBHISTORY_BLOCK_SIZE = - "mapreduce.jobtracker.jobhistory.block.size"; - public static final String JT_JOBHISTORY_COMPLETED_LOCATION = - "mapreduce.jobtracker.jobhistory.completed.location"; - public static final String JT_JOBHISTORY_LOCATION = - "mapreduce.jobtracker.jobhistory.location"; - // number of partial task progress reports we retain in job history - public static final String JT_JOBHISTORY_TASKPROGRESS_NUMBER_SPLITS = - "mapreduce.jobtracker.jobhistory.task.numberprogresssplits"; - public static final String JT_AVG_BLACKLIST_THRESHOLD = - "mapreduce.jobtracker.blacklist.average.threshold"; public static final String JT_SYSTEM_DIR = "mapreduce.jobtracker.system.dir"; public static final String JT_STAGING_AREA_ROOT = "mapreduce.jobtracker.staging.root.dir"; - public static final String JT_MAX_TRACKER_BLACKLISTS = - "mapreduce.jobtracker.tasktracker.maxblacklists"; - public static final String JT_JOBHISTORY_MAXAGE = - "mapreduce.jobtracker.jobhistory.maxage"; - public static final String JT_MAX_MAPMEMORY_MB = + public static final String JT_MAX_MAPMEMORY_MB = "mapreduce.jobtracker.maxmapmemory.mb"; public static final String JT_MAX_REDUCEMEMORY_MB = "mapreduce.jobtracker.maxreducememory.mb"; - public static final String JT_MAX_JOB_SPLIT_METAINFO_SIZE = - "mapreduce.jobtracker.split.metainfo.maxsize"; public static final String JT_USER_NAME = "mapreduce.jobtracker.kerberos.principal"; - public static final String JT_KEYTAB_FILE = - "mapreduce.jobtracker.keytab.file"; - public static final String PRIVATE_ACTIONS_KEY = - "mapreduce.jobtracker.webinterface.trusted"; - public static final String JT_PLUGINS = - "mapreduce.jobtracker.plugins"; - public static final String SHUFFLE_EXCEPTION_STACK_REGEX = - "mapreduce.reduce.shuffle.catch.exception.stack.regex"; - public static final String SHUFFLE_EXCEPTION_MSG_REGEX = - "mapreduce.reduce.shuffle.catch.exception.message.regex"; - } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java index 66b630a69f..d3f62e4c93 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java @@ -68,64 +68,16 @@ private static void addDeprecatedKeys() { new DeprecationDelta("mapred.cluster.max.reduce.memory.mb", JTConfig.JT_MAX_REDUCEMEMORY_MB), - new DeprecationDelta("mapred.cluster.average.blacklist.threshold", - JTConfig.JT_AVG_BLACKLIST_THRESHOLD), - new DeprecationDelta("hadoop.job.history.location", - JTConfig.JT_JOBHISTORY_LOCATION), - new DeprecationDelta( - "mapred.job.tracker.history.completed.location", - JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION), - new DeprecationDelta("mapred.jobtracker.job.history.block.size", - JTConfig.JT_JOBHISTORY_BLOCK_SIZE), - new DeprecationDelta("mapred.job.tracker.jobhistory.lru.cache.size", - JTConfig.JT_JOBHISTORY_CACHE_SIZE), - new DeprecationDelta("mapred.hosts", - JTConfig.JT_HOSTS_FILENAME), - new DeprecationDelta("mapred.hosts.exclude", - JTConfig.JT_HOSTS_EXCLUDE_FILENAME), new DeprecationDelta("mapred.system.dir", JTConfig.JT_SYSTEM_DIR), - new DeprecationDelta("mapred.max.tracker.blacklists", - JTConfig.JT_MAX_TRACKER_BLACKLISTS), new DeprecationDelta("mapred.job.tracker", JTConfig.JT_IPC_ADDRESS), - new DeprecationDelta("mapred.job.tracker.http.address", - JTConfig.JT_HTTP_ADDRESS), - new DeprecationDelta("mapred.job.tracker.handler.count", - JTConfig.JT_IPC_HANDLER_COUNT), - new DeprecationDelta("mapred.jobtracker.restart.recover", - JTConfig.JT_RESTART_ENABLED), - new DeprecationDelta("mapred.jobtracker.taskScheduler", - JTConfig.JT_TASK_SCHEDULER), - new DeprecationDelta( - "mapred.jobtracker.taskScheduler.maxRunningTasksPerJob", - JTConfig.JT_RUNNINGTASKS_PER_JOB), - new DeprecationDelta("mapred.jobtracker.instrumentation", - JTConfig.JT_INSTRUMENTATION), - new DeprecationDelta("mapred.jobtracker.maxtasks.per.job", - JTConfig.JT_TASKS_PER_JOB), - new DeprecationDelta("mapred.heartbeats.in.second", - JTConfig.JT_HEARTBEATS_IN_SECOND), new DeprecationDelta("mapred.job.tracker.persist.jobstatus.active", JTConfig.JT_PERSIST_JOBSTATUS), - new DeprecationDelta("mapred.job.tracker.persist.jobstatus.hours", - JTConfig.JT_PERSIST_JOBSTATUS_HOURS), - new DeprecationDelta("mapred.job.tracker.persist.jobstatus.dir", - JTConfig.JT_PERSIST_JOBSTATUS_DIR), new DeprecationDelta("mapred.permissions.supergroup", MRConfig.MR_SUPERGROUP), - new DeprecationDelta("mapreduce.jobtracker.permissions.supergroup", - MRConfig.MR_SUPERGROUP), new DeprecationDelta("mapred.task.cache.levels", JTConfig.JT_TASKCACHE_LEVELS), - new DeprecationDelta("mapred.jobtracker.taskalloc.capacitypad", - JTConfig.JT_TASK_ALLOC_PAD_FRACTION), - new DeprecationDelta("mapred.jobinit.threads", - JTConfig.JT_JOBINIT_THREADS), - new DeprecationDelta("mapred.tasktracker.expiry.interval", - JTConfig.JT_TRACKER_EXPIRY_INTERVAL), - new DeprecationDelta("mapred.job.tracker.retiredjobs.cache.size", - JTConfig.JT_RETIREJOB_CACHE_SIZE), new DeprecationDelta("mapred.job.tracker.retire.jobs", JTConfig.JT_RETIREJOBS), new DeprecationDelta("mapred.healthChecker.interval", @@ -511,17 +463,12 @@ private static void addDeprecatedKeys() { new DeprecationDelta("mapred.pipes.user.inputformat", org.apache.hadoop.mapred.pipes.Submitter.INPUT_FORMAT), - new DeprecationDelta("webinterface.private.actions", - JTConfig.PRIVATE_ACTIONS_KEY), - new DeprecationDelta("security.task.umbilical.protocol.acl", MRJobConfig.MR_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL), new DeprecationDelta("security.job.submission.protocol.acl", MRJobConfig.MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT), new DeprecationDelta("mapreduce.user.classpath.first", MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST ), - new DeprecationDelta(JTConfig.JT_MAX_JOB_SPLIT_METAINFO_SIZE, - MRJobConfig.SPLIT_METAINFO_MAXSIZE), new DeprecationDelta("mapred.input.dir.recursive", FileInputFormat.INPUT_DIR_RECURSIVE) }); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml index cbae524b18..e4619d6792 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml @@ -46,7 +46,6 @@ dfs.namenode.edits.dir${dfs.namenode.name.dir} mapreduce.job.map.classorg.apache.hadoop.mapreduce.SleepJob$SleepMapper dfs.datanode.http.address0.0.0.0:9864 -mapreduce.jobtracker.jobhistory.task.numberprogresssplits12 yarn.acl.enabletrue yarn.nodemanager.localizer.fetch.thread-count4 hadoop.proxyuser.user.hosts127.0.0.1 @@ -55,7 +54,6 @@ mapreduce.reduce.log.levelINFO yarn.log-aggregation-enablefalse dfs.https.server.keystore.resourcessl-server.xml -mapreduce.jobtracker.instrumentationorg.apache.hadoop.mapred.JobTrackerMetricsInst dfs.namenode.replication.min1 mapreduce.map.java.opts-Xmx500m s3native.bytes-per-checksum512 @@ -72,7 +70,6 @@ dfs.namenode.max.objects0 dfs.namenode.safemode.threshold-pct0.999f mapreduce.tasktracker.local.dir.minspacekill0 -mapreduce.jobtracker.retiredjobs.cache.size1000 dfs.blocksize67108864 mapreduce.job.reduce.slowstart.completedmaps0.05 mapreduce.job.end-notification.retry.attempts5 @@ -82,7 +79,6 @@ mapreduce.job.user.nameuser mapreduce.tasktracker.outofband.heartbeatfalse io.native.lib.availabletrue -mapreduce.jobtracker.persist.jobstatus.hours1 dfs.client-write-packet-size65536 mapreduce.client.progressmonitor.pollinterval1000 dfs.namenode.name.dirfile:///home/user/hadoop-user/dfs/name @@ -98,7 +94,6 @@ mapreduce.tasktracker.taskcontrollerorg.apache.hadoop.mapred.DefaultTaskController mapreduce.reduce.shuffle.parallelcopies5 yarn.nodemanager.env-whitelistJAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME -mapreduce.jobtracker.heartbeats.in.second100 mapreduce.job.maxtaskfailures.per.tracker4 ipc.client.connection.maxidletime10000 dfs.blockreport.intervalMsec21600000 @@ -114,7 +109,6 @@ ipc.client.tcpnodelayfalse mapreduce.map.output.value.classorg.apache.hadoop.io.NullWritable dfs.namenode.accesstime.precision3600000 -mapreduce.jobtracker.tasktracker.maxblacklists4 rpc.engine.com.google.protobuf.BlockingServiceorg.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine mapreduce.job.jvm.numtasks1 mapreduce.task.io.sort.mb100 @@ -133,7 +127,6 @@ fs.har.implorg.apache.hadoop.fs.HarFileSystem hadoop.security.authenticationsimple fs.s3n.buffer.dir${hadoop.tmp.dir}/s3n -mapreduce.jobtracker.taskschedulerorg.apache.hadoop.mapred.JobQueueTaskScheduler yarn.app.mapreduce.am.job.task.listener.thread-count30 mapreduce.job.reduces1 mapreduce.map.sort.spill.percent0.80 @@ -154,7 +147,6 @@ fs.file.implorg.apache.hadoop.fs.LocalFileSystem dfs.namenode.https-address0.0.0.0:9871 mapreduce.task.skip.start.attempts2 -mapreduce.jobtracker.persist.jobstatus.dir/jobtracker/jobsInfo ipc.client.kill.max10 mapreduce.job.end-notification.max.attempts5 mapreduce.jobhistory.max-age-ms10000000000 @@ -175,9 +167,7 @@ io.mapfile.bloom.size1048576 yarn.nodemanager.container-executor.classorg.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor mapreduce.map.maxattempts4 -mapreduce.jobtracker.jobhistory.block.size3145728 ftp.replication3 -mapreduce.jobtracker.http.address0.0.0.0:50030 yarn.nodemanager.health-checker.script.timeout-ms1200000 mapreduce.jobhistory.address0.0.0.0:10020 dfs.datanode.dns.nameserverdefault @@ -317,7 +307,6 @@ dfs.datanode.https.address0.0.0.0:9865 file.replication1 yarn.resourcemanager.resource-tracker.address0.0.0.0:8025 -mapreduce.jobtracker.restart.recoverfalse hadoop.work.around.non.threadsafe.getpwuidfalse mapreduce.client.genericoptionsparser.usedtrue mapreduce.tasktracker.indexcache.mb10 @@ -331,7 +320,6 @@ file.bytes-per-checksum512 mapreduce.input.fileinputformat.inputdirhdfs://localhost:8021/user/user/ignored dfs.client.block.write.replace-datanode-on-failure.enableture -mapreduce.jobtracker.handler.count10 net.topology.script.number.args100 mapreduce.task.profile.maps0-2 dfs.namenode.decommission.interval30 @@ -342,7 +330,6 @@ yarn.nodemanager.vmem-pmem-ratio2.1 io.mapfile.bloom.error.rate0.005 dfs.permissions.superusergroupsupergroup -mapreduce.jobtracker.expire.trackers.interval600000 mapreduce.cluster.acls.enabledfalse yarn.nodemanager.remote-app-log-dir-suffixlogs dfs.namenode.checkpoint.check.period60 @@ -351,8 +338,6 @@ mapreduce.reduce.shuffle.merge.percent0.66 tfile.io.chunk.size1048576 file.blocksize67108864 -mapreduce.jobtracker.jobhistory.lru.cache.size5 -mapreduce.jobtracker.maxtasks.perjob-1 yarn.resourcemanager.nm.liveness-monitor.interval-ms1000 yarn.nodemanager.webapp.address0.0.0.0:9999 mapreduce.job.acl-modify-job diff --git a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json index cc3dcea95a..095cfd5732 100644 --- a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json +++ b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json @@ -1,4 +1,4 @@ -{ +[{ "priority" : "NORMAL", "jobID" : "job_1369942127770_1205", "user" : "jenkins", @@ -4585,7 +4585,6 @@ "dfs.datanode.http.address" : "0.0.0.0:9864", "mapreduce.terasort.num-rows" : "400000000", "mapreduce.job.map.class" : "org.apache.hadoop.examples.terasort.TeraGen$SortGenMapper", - "mapreduce.jobtracker.jobhistory.task.numberprogresssplits" : "12", "dfs.namenode.write.stale.datanode.ratio" : "0.5f", "dfs.client.use.datanode.hostname" : "false", "yarn.acl.enable" : "true", @@ -4602,7 +4601,6 @@ "mapreduce.reduce.log.level" : "INFO", "yarn.log-aggregation-enable" : "false", "dfs.datanode.sync.behind.writes" : "false", - "mapreduce.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst", "dfs.https.server.keystore.resource" : "ssl-server.xml", "hadoop.security.group.mapping.ldap.search.attr.group.name" : "cn", "dfs.namenode.replication.min" : "1", @@ -4625,7 +4623,6 @@ "hadoop.ssl.client.conf" : "ssl-client.xml", "dfs.namenode.safemode.threshold-pct" : "0.999f", "mapreduce.tasktracker.local.dir.minspacekill" : "0", - "mapreduce.jobtracker.retiredjobs.cache.size" : "1000", "dfs.blocksize" : "134217728", "yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler", "mapreduce.job.reduce.slowstart.completedmaps" : "0.8", @@ -4635,7 +4632,6 @@ "mapreduce.job.user.name" : "jenkins", "mapreduce.tasktracker.outofband.heartbeat" : "false", "io.native.lib.available" : "true", - "mapreduce.jobtracker.persist.jobstatus.hours" : "0", "dfs.client-write-packet-size" : "65536", "mapreduce.client.progressmonitor.pollinterval" : "1000", "dfs.namenode.name.dir" : "file://${hadoop.tmp.dir}/dfs/name", @@ -4653,7 +4649,6 @@ "yarn.scheduler.fair.preemption" : "true", "mapreduce.reduce.shuffle.parallelcopies" : "5", "yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME", - "mapreduce.jobtracker.heartbeats.in.second" : "100", "mapreduce.job.maxtaskfailures.per.tracker" : "3", "ipc.client.connection.maxidletime" : "10000", "mapreduce.shuffle.ssl.enabled" : "false", @@ -4673,7 +4668,6 @@ "dfs.replication" : "3", "ipc.client.tcpnodelay" : "false", "dfs.namenode.accesstime.precision" : "3600000", - "mapreduce.jobtracker.tasktracker.maxblacklists" : "4", "dfs.client.read.shortcircuit.skip.checksum" : "false", "mapreduce.job.jvm.numtasks" : "1", "mapreduce.task.io.sort.mb" : "100", @@ -4692,7 +4686,6 @@ "hadoop.security.authentication" : "simple", "fs.s3n.buffer.dir" : "${hadoop.tmp.dir}/s3n", "rpc.engine.org.apache.hadoop.yarn.api.AMRMProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine", - "mapreduce.jobtracker.taskscheduler" : "org.apache.hadoop.mapred.JobQueueTaskScheduler", "yarn.app.mapreduce.am.job.task.listener.thread-count" : "30", "dfs.namenode.avoid.read.stale.datanode" : "false", "mapreduce.job.reduces" : "0", @@ -4718,7 +4711,6 @@ "dfs.namenode.replication.interval" : "3", "mapreduce.task.skip.start.attempts" : "2", "dfs.namenode.https-address" : "a2115.smile.com:20102", - "mapreduce.jobtracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo", "ipc.client.kill.max" : "10", "dfs.ha.automatic-failover.enabled" : "false", "mapreduce.jobhistory.keytab" : "/etc/security/keytab/jhs.service.keytab", @@ -4741,11 +4733,9 @@ "io.mapfile.bloom.size" : "1048576", "yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor", "mapreduce.map.maxattempts" : "4", - "mapreduce.jobtracker.jobhistory.block.size" : "3145728", "yarn.log-aggregation.retain-seconds" : "-1", "yarn.app.mapreduce.am.job.committer.cancel-timeout" : "60000", "ftp.replication" : "3", - "mapreduce.jobtracker.http.address" : "0.0.0.0:50030", "yarn.nodemanager.health-checker.script.timeout-ms" : "1200000", "mapreduce.jobhistory.address" : "a2115.smile.com:10020", "mapreduce.jobtracker.taskcache.levels" : "2", @@ -4903,7 +4893,6 @@ "dfs.datanode.drop.cache.behind.reads" : "false", "hadoop.fuse.connection.timeout" : "300", "hadoop.work.around.non.threadsafe.getpwuid" : "false", - "mapreduce.jobtracker.restart.recover" : "false", "hadoop.tmp.dir" : "/tmp/hadoop-${user.name}", "mapreduce.output.fileoutputformat.compress" : "false", "mapreduce.tasktracker.indexcache.mb" : "10", @@ -4915,7 +4904,6 @@ "dfs.namenode.fs-limits.max-blocks-per-file" : "1048576", "file.bytes-per-checksum" : "512", "mapreduce.task.profile.reduces" : "0-2", - "mapreduce.jobtracker.handler.count" : "10", "dfs.client.block.write.replace-datanode-on-failure.enable" : "true", "mapreduce.job.output.value.class" : "org.apache.hadoop.io.Text", "yarn.dispatcher.exit-on-error" : "true", @@ -4933,7 +4921,6 @@ "mapreduce.shuffle.ssl.file.buffer.size" : "65536", "dfs.permissions.superusergroup" : "supergroup", "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold" : "10737418240", - "mapreduce.jobtracker.expire.trackers.interval" : "600000", "mapreduce.cluster.acls.enabled" : "false", "yarn.nodemanager.remote-app-log-dir-suffix" : "logs", "ha.failover-controller.graceful-fence.connection.retries" : "1", @@ -4943,8 +4930,6 @@ "io.seqfile.local.dir" : "${hadoop.tmp.dir}/io/local", "dfs.namenode.checkpoint.check.period" : "60", "yarn.resourcemanager.nm.liveness-monitor.interval-ms" : "1000", - "mapreduce.jobtracker.maxtasks.perjob" : "-1", - "mapreduce.jobtracker.jobhistory.lru.cache.size" : "5", "file.blocksize" : "67108864", "tfile.io.chunk.size" : "1048576", "mapreduce.job.acl-modify-job" : " ", @@ -5095,7 +5080,7 @@ "clusterReduceMB" : -1, "jobMapMB" : 200, "jobReduceMB" : 200 -} { +}, { "priority" : "NORMAL", "jobID" : "job_1369942127770_1206", "user" : "jenkins", @@ -9682,7 +9667,6 @@ "dfs.datanode.http.address" : "0.0.0.0:9864", "mapreduce.terasort.num-rows" : "400000000", "mapreduce.job.map.class" : "org.apache.hadoop.examples.terasort.TeraGen$SortGenMapper", - "mapreduce.jobtracker.jobhistory.task.numberprogresssplits" : "12", "dfs.namenode.write.stale.datanode.ratio" : "0.5f", "dfs.client.use.datanode.hostname" : "false", "yarn.acl.enable" : "true", @@ -9699,7 +9683,6 @@ "mapreduce.reduce.log.level" : "INFO", "yarn.log-aggregation-enable" : "false", "dfs.datanode.sync.behind.writes" : "false", - "mapreduce.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst", "dfs.https.server.keystore.resource" : "ssl-server.xml", "hadoop.security.group.mapping.ldap.search.attr.group.name" : "cn", "dfs.namenode.replication.min" : "1", @@ -9722,7 +9705,6 @@ "hadoop.ssl.client.conf" : "ssl-client.xml", "dfs.namenode.safemode.threshold-pct" : "0.999f", "mapreduce.tasktracker.local.dir.minspacekill" : "0", - "mapreduce.jobtracker.retiredjobs.cache.size" : "1000", "dfs.blocksize" : "134217728", "yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler", "mapreduce.job.reduce.slowstart.completedmaps" : "0.8", @@ -9732,7 +9714,6 @@ "mapreduce.job.user.name" : "jenkins", "mapreduce.tasktracker.outofband.heartbeat" : "false", "io.native.lib.available" : "true", - "mapreduce.jobtracker.persist.jobstatus.hours" : "0", "dfs.client-write-packet-size" : "65536", "mapreduce.client.progressmonitor.pollinterval" : "1000", "dfs.namenode.name.dir" : "file://${hadoop.tmp.dir}/dfs/name", @@ -9750,7 +9731,6 @@ "yarn.scheduler.fair.preemption" : "true", "mapreduce.reduce.shuffle.parallelcopies" : "5", "yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME", - "mapreduce.jobtracker.heartbeats.in.second" : "100", "mapreduce.job.maxtaskfailures.per.tracker" : "3", "ipc.client.connection.maxidletime" : "10000", "mapreduce.shuffle.ssl.enabled" : "false", @@ -9770,7 +9750,6 @@ "dfs.replication" : "3", "ipc.client.tcpnodelay" : "false", "dfs.namenode.accesstime.precision" : "3600000", - "mapreduce.jobtracker.tasktracker.maxblacklists" : "4", "dfs.client.read.shortcircuit.skip.checksum" : "false", "mapreduce.job.jvm.numtasks" : "1", "mapreduce.task.io.sort.mb" : "100", @@ -9789,7 +9768,6 @@ "hadoop.security.authentication" : "simple", "fs.s3n.buffer.dir" : "${hadoop.tmp.dir}/s3", "rpc.engine.org.apache.hadoop.yarn.api.AMRMProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine", - "mapreduce.jobtracker.taskscheduler" : "org.apache.hadoop.mapred.JobQueueTaskScheduler", "yarn.app.mapreduce.am.job.task.listener.thread-count" : "30", "dfs.namenode.avoid.read.stale.datanode" : "false", "mapreduce.job.reduces" : "0", @@ -9815,7 +9793,6 @@ "dfs.namenode.replication.interval" : "3", "mapreduce.task.skip.start.attempts" : "2", "dfs.namenode.https-address" : "a2115.smile.com:20102", - "mapreduce.jobtracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo", "ipc.client.kill.max" : "10", "dfs.ha.automatic-failover.enabled" : "false", "mapreduce.jobhistory.keytab" : "/etc/security/keytab/jhs.service.keytab", @@ -9838,11 +9815,9 @@ "io.mapfile.bloom.size" : "1048576", "yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor", "mapreduce.map.maxattempts" : "4", - "mapreduce.jobtracker.jobhistory.block.size" : "3145728", "yarn.log-aggregation.retain-seconds" : "-1", "yarn.app.mapreduce.am.job.committer.cancel-timeout" : "60000", "ftp.replication" : "3", - "mapreduce.jobtracker.http.address" : "0.0.0.0:50030", "yarn.nodemanager.health-checker.script.timeout-ms" : "1200000", "mapreduce.jobhistory.address" : "a2115.smile.com:10020", "mapreduce.jobtracker.taskcache.levels" : "2", @@ -10000,7 +9975,6 @@ "dfs.datanode.drop.cache.behind.reads" : "false", "hadoop.fuse.connection.timeout" : "300", "hadoop.work.around.non.threadsafe.getpwuid" : "false", - "mapreduce.jobtracker.restart.recover" : "false", "hadoop.tmp.dir" : "/tmp/hadoop-${user.name}", "mapreduce.output.fileoutputformat.compress" : "false", "mapreduce.tasktracker.indexcache.mb" : "10", @@ -10012,7 +9986,6 @@ "dfs.namenode.fs-limits.max-blocks-per-file" : "1048576", "file.bytes-per-checksum" : "512", "mapreduce.task.profile.reduces" : "0-2", - "mapreduce.jobtracker.handler.count" : "10", "dfs.client.block.write.replace-datanode-on-failure.enable" : "true", "mapreduce.job.output.value.class" : "org.apache.hadoop.io.Text", "yarn.dispatcher.exit-on-error" : "true", @@ -10030,7 +10003,6 @@ "mapreduce.shuffle.ssl.file.buffer.size" : "65536", "dfs.permissions.superusergroup" : "supergroup", "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold" : "10737418240", - "mapreduce.jobtracker.expire.trackers.interval" : "600000", "mapreduce.cluster.acls.enabled" : "false", "yarn.nodemanager.remote-app-log-dir-suffix" : "logs", "ha.failover-controller.graceful-fence.connection.retries" : "1", @@ -10040,8 +10012,6 @@ "io.seqfile.local.dir" : "${hadoop.tmp.dir}/io/local", "dfs.namenode.checkpoint.check.period" : "60", "yarn.resourcemanager.nm.liveness-monitor.interval-ms" : "1000", - "mapreduce.jobtracker.maxtasks.perjob" : "-1", - "mapreduce.jobtracker.jobhistory.lru.cache.size" : "5", "file.blocksize" : "67108864", "tfile.io.chunk.size" : "1048576", "mapreduce.job.acl-modify-job" : " ", @@ -10192,7 +10162,7 @@ "clusterReduceMB" : -1, "jobMapMB" : 200, "jobReduceMB" : 200 -} { +}, { "priority" : "NORMAL", "jobID" : "job_1369942127770_1207", "user" : "jenkins", @@ -10279,7 +10249,6 @@ "dfs.datanode.http.address" : "0.0.0.0:9864", "mapreduce.terasort.num-rows" : "400000000", "mapreduce.job.map.class" : "org.apache.hadoop.examples.terasort.TeraGen$SortGenMapper", -"mapreduce.jobtracker.jobhistory.task.numberprogresssplits" : "12", "dfs.namenode.write.stale.datanode.ratio" : "0.5f", "dfs.client.use.datanode.hostname" : "false", "yarn.acl.enable" : "true", @@ -10296,7 +10265,6 @@ "mapreduce.reduce.log.level" : "INFO", "yarn.log-aggregation-enable" : "false", "dfs.datanode.sync.behind.writes" : "false", -"mapreduce.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst", "dfs.https.server.keystore.resource" : "ssl-server.xml", "hadoop.security.group.mapping.ldap.search.attr.group.name" : "cn", "dfs.namenode.replication.min" : "1", @@ -10319,7 +10287,6 @@ "hadoop.ssl.client.conf" : "ssl-client.xml", "dfs.namenode.safemode.threshold-pct" : "0.999f", "mapreduce.tasktracker.local.dir.minspacekill" : "0", -"mapreduce.jobtracker.retiredjobs.cache.size" : "1000", "dfs.blocksize" : "134217728", "yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler", "mapreduce.job.reduce.slowstart.completedmaps" : "0.8", @@ -10329,7 +10296,6 @@ "mapreduce.job.user.name" : "jenkins", "mapreduce.tasktracker.outofband.heartbeat" : "false", "io.native.lib.available" : "true", -"mapreduce.jobtracker.persist.jobstatus.hours" : "0", "dfs.client-write-packet-size" : "65536", "mapreduce.client.progressmonitor.pollinterval" : "1000", "dfs.namenode.name.dir" : "file://${hadoop.tmp.dir}/dfs/name", @@ -10348,7 +10314,6 @@ "mapreduce.reduce.shuffle.parallelcopies" : "5", "dfs.support.append" : "true", "yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME", -"mapreduce.jobtracker.heartbeats.in.second" : "100", "mapreduce.job.maxtaskfailures.per.tracker" : "3", "ipc.client.connection.maxidletime" : "10000", "mapreduce.shuffle.ssl.enabled" : "false", @@ -10368,7 +10333,6 @@ "dfs.replication" : "3", "ipc.client.tcpnodelay" : "false", "dfs.namenode.accesstime.precision" : "3600000", -"mapreduce.jobtracker.tasktracker.maxblacklists" : "4", "dfs.client.read.shortcircuit.skip.checksum" : "false", "mapreduce.job.jvm.numtasks" : "1", "mapreduce.task.io.sort.mb" : "100", @@ -10387,7 +10351,6 @@ "hadoop.security.authentication" : "simple", "fs.s3n.buffer.dir" : "${hadoop.tmp.dir}/s3n", "rpc.engine.org.apache.hadoop.yarn.api.AMRMProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine", -"mapreduce.jobtracker.taskscheduler" : "org.apache.hadoop.mapred.JobQueueTaskScheduler", "yarn.app.mapreduce.am.job.task.listener.thread-count" : "30", "dfs.namenode.avoid.read.stale.datanode" : "false", "mapreduce.job.reduces" : "0", @@ -10413,7 +10376,6 @@ "dfs.namenode.replication.interval" : "3", "mapreduce.task.skip.start.attempts" : "2", "dfs.namenode.https-address" : "a2115.smile.com:20102", -"mapreduce.jobtracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo", "ipc.client.kill.max" : "10", "dfs.ha.automatic-failover.enabled" : "false", "mapreduce.jobhistory.keytab" : "/etc/security/keytab/jhs.service.keytab", @@ -10436,11 +10398,9 @@ "io.mapfile.bloom.size" : "1048576", "yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor", "mapreduce.map.maxattempts" : "4", -"mapreduce.jobtracker.jobhistory.block.size" : "3145728", "yarn.log-aggregation.retain-seconds" : "-1", "yarn.app.mapreduce.am.job.committer.cancel-timeout" : "60000", "ftp.replication" : "3", -"mapreduce.jobtracker.http.address" : "0.0.0.0:50030", "yarn.nodemanager.health-checker.script.timeout-ms" : "1200000", "mapreduce.jobhistory.address" : "a2115.smile.com:10020", "mapreduce.jobtracker.taskcache.levels" : "2", @@ -10598,7 +10558,6 @@ "dfs.datanode.drop.cache.behind.reads" : "false", "hadoop.fuse.connection.timeout" : "300", "hadoop.work.around.non.threadsafe.getpwuid" : "false", -"mapreduce.jobtracker.restart.recover" : "false", "hadoop.tmp.dir" : "/tmp/hadoop-${user.name}", "mapreduce.output.fileoutputformat.compress" : "false", "mapreduce.tasktracker.indexcache.mb" : "10", @@ -10610,7 +10569,6 @@ "dfs.namenode.fs-limits.max-blocks-per-file" : "1048576", "file.bytes-per-checksum" : "512", "mapreduce.task.profile.reduces" : "0-2", -"mapreduce.jobtracker.handler.count" : "10", "dfs.client.block.write.replace-datanode-on-failure.enable" : "true", "mapreduce.job.output.value.class" : "org.apache.hadoop.io.Text", "yarn.dispatcher.exit-on-error" : "true", @@ -10628,7 +10586,6 @@ "mapreduce.shuffle.ssl.file.buffer.size" : "65536", "dfs.permissions.superusergroup" : "supergroup", "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold" : "10737418240", -"mapreduce.jobtracker.expire.trackers.interval" : "600000", "mapreduce.cluster.acls.enabled" : "false", "yarn.nodemanager.remote-app-log-dir-suffix" : "logs", "ha.failover-controller.graceful-fence.connection.retries" : "1", @@ -10638,8 +10595,6 @@ "io.seqfile.local.dir" : "${hadoop.tmp.dir}/io/local", "dfs.namenode.checkpoint.check.period" : "60", "yarn.resourcemanager.nm.liveness-monitor.interval-ms" : "1000", -"mapreduce.jobtracker.maxtasks.perjob" : "-1", -"mapreduce.jobtracker.jobhistory.lru.cache.size" : "5", "file.blocksize" : "67108864", "tfile.io.chunk.size" : "1048576", "mapreduce.job.acl-modify-job" : " ", @@ -10791,3 +10746,4 @@ "jobMapMB" : 200, "jobReduceMB" : 200 } +]