big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6794. Remove unused properties from TTConfig.java

Browse Source
This commit is contained in:
Akira Ajisaka 2019-06-06 16:36:34 +09:00
parent cb9bc6e64c
commit 3ea4f41d9f
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
5 changed files with 2 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
View File

@ -65,7 +65,6 @@ The following table lists the configuration property names that are deprecated i
| fs.s3a.server-side-encryption-key | fs.s3a.server-side-encryption.key |
| hadoop.configured.node.mapping | net.topology.configured.node.mapping |
| hadoop.native.lib | io.native.lib.available |
| hadoop.net.static.resolutions | mapreduce.tasktracker.net.static.resolutions |
| hadoop.pipes.command-file.keep | mapreduce.pipes.commandfile.preserve |
| hadoop.pipes.executable.interpretor | mapreduce.pipes.executable.interpretor |
| hadoop.pipes.executable | mapreduce.pipes.executable |
@ -89,7 +88,6 @@ The following table lists the configuration property names that are deprecated i
| keep.failed.task.files | mapreduce.task.files.preserve.failedtasks |
| keep.task.files.pattern | mapreduce.task.files.preserve.filepattern |
| key.value.separator.in.input.line | mapreduce.input.keyvaluelinerecordreader.key.value.separator |
| local.cache.size | mapreduce.tasktracker.cache.local.size |
| map.input.file | mapreduce.map.input.file |
| map.input.length | mapreduce.map.input.length |
| map.input.start | mapreduce.map.input.start |
@ -113,10 +111,6 @@ The following table lists the configuration property names that are deprecated i
| mapred.compress.map.output | mapreduce.map.output.compress |
| mapred.data.field.separator | mapreduce.fieldsel.data.field.separator |
| mapred.debug.out.lines | mapreduce.task.debugout.lines |
| mapred.healthChecker.interval | mapreduce.tasktracker.healthchecker.interval |
| mapred.healthChecker.script.args | mapreduce.tasktracker.healthchecker.script.args |
| mapred.healthChecker.script.path | mapreduce.tasktracker.healthchecker.script.path |
| mapred.healthChecker.script.timeout | mapreduce.tasktracker.healthchecker.script.timeout |
| mapred.inmem.merge.threshold | mapreduce.reduce.merge.inmem.threshold |
| mapred.input.dir.formats | mapreduce.input.multipleinputs.dir.formats |
| mapred.input.dir.mappers | mapreduce.input.multipleinputs.dir.mappers |
@ -146,8 +140,6 @@ The following table lists the configuration property names that are deprecated i
| mapred.line.input.format.linespermap | mapreduce.input.lineinputformat.linespermap |
| mapred.linerecordreader.maxlength | mapreduce.input.linerecordreader.line.maxlength |
| mapred.local.dir | mapreduce.cluster.local.dir |
| mapred.local.dir.minspacekill | mapreduce.tasktracker.local.dir.minspacekill |
| mapred.local.dir.minspacestart | mapreduce.tasktracker.local.dir.minspacestart |
| mapred.map.child.env | mapreduce.map.env |
| mapred.map.child.java.opts | mapreduce.map.java.opts |
| mapred.map.child.log.level | mapreduce.map.log.level |
@ -212,19 +204,10 @@ The following table lists the configuration property names that are deprecated i
| mapred.task.profile.params | mapreduce.task.profile.params |
| mapred.task.profile.reduces | mapreduce.task.profile.reduces |
| mapred.task.timeout | mapreduce.task.timeout |
| mapred.tasktracker.dns.interface | mapreduce.tasktracker.dns.interface |
| mapred.tasktracker.dns.nameserver | mapreduce.tasktracker.dns.nameserver |
| mapred.tasktracker.events.batchsize | mapreduce.tasktracker.events.batchsize |
| mapred.task.tracker.http.address | mapreduce.tasktracker.http.address |
| mapred.tasktracker.indexcache.mb | mapreduce.tasktracker.indexcache.mb |
| mapred.tasktracker.instrumentation | mapreduce.tasktracker.instrumentation |
| mapred.tasktracker.map.tasks.maximum | mapreduce.tasktracker.map.tasks.maximum |
| mapred.tasktracker.memory\_calculator\_plugin | mapreduce.tasktracker.resourcecalculatorplugin |
| mapred.tasktracker.memorycalculatorplugin | mapreduce.tasktracker.resourcecalculatorplugin |
| mapred.tasktracker.reduce.tasks.maximum | mapreduce.tasktracker.reduce.tasks.maximum |
| mapred.task.tracker.report.address | mapreduce.tasktracker.report.address |
| mapred.task.tracker.task-controller | mapreduce.tasktracker.taskcontroller |
| mapred.tasktracker.tasks.sleeptime-before-sigkill | mapreduce.tasktracker.tasks.sleeptimebeforesigkill |
| mapred.temp.dir | mapreduce.cluster.temp.dir |
| mapred.text.key.comparator.options | mapreduce.partition.keycomparator.options |
| mapred.text.key.partitioner.options | mapreduce.partition.keypartitioner.options |
@ -251,9 +234,6 @@ The following table lists the configuration property names that are deprecated i
| sequencefile.filter.regex | mapreduce.input.sequencefileinputfilter.regex |
| session.id | dfs.metrics.session-id |
| slave.host.name | dfs.datanode.hostname |
| slave.host.name | mapreduce.tasktracker.host.name |
| tasktracker.contention.tracking | mapreduce.tasktracker.contention.tracking |
| tasktracker.http.threads | mapreduce.tasktracker.http.threads |
| topology.node.switch.mapping.impl | net.topology.node.switch.mapping.impl |
| topology.script.file.name | net.topology.script.file.name |
| topology.script.number.args | net.topology.script.number.args |

73
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
View File

@ -22,86 +22,17 @@
import org.apache.hadoop.mapreduce.MRConfig;
/**
* Place holder for TaskTracker server-level configuration.
*
* The keys should have "mapreduce.tasktracker." as the prefix
* Place holder for MapReduce server-level configuration.
* (formerly TaskTracker configuration)
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public interface TTConfig extends MRConfig {
// Task-tracker configuration properties
public static final String TT_HEALTH_CHECKER_INTERVAL =
"mapreduce.tasktracker.healthchecker.interval";
public static final String TT_HEALTH_CHECKER_SCRIPT_ARGS =
"mapreduce.tasktracker.healthchecker.script.args";
public static final String TT_HEALTH_CHECKER_SCRIPT_PATH =
"mapreduce.tasktracker.healthchecker.script.path";
public static final String TT_HEALTH_CHECKER_SCRIPT_TIMEOUT =
"mapreduce.tasktracker.healthchecker.script.timeout";
public static final String TT_LOCAL_DIR_MINSPACE_KILL =
"mapreduce.tasktracker.local.dir.minspacekill";
public static final String TT_LOCAL_DIR_MINSPACE_START =
"mapreduce.tasktracker.local.dir.minspacestart";
public static final String TT_HTTP_ADDRESS =
"mapreduce.tasktracker.http.address";
public static final String TT_REPORT_ADDRESS =
"mapreduce.tasktracker.report.address";
public static final String TT_TASK_CONTROLLER =
"mapreduce.tasktracker.taskcontroller";
public static final String TT_CONTENTION_TRACKING =
"mapreduce.tasktracker.contention.tracking";
public static final String TT_STATIC_RESOLUTIONS =
"mapreduce.tasktracker.net.static.resolutions";
public static final String TT_HTTP_THREADS =
"mapreduce.tasktracker.http.threads";
public static final String TT_HOST_NAME = "mapreduce.tasktracker.host.name";
public static final String TT_SLEEP_TIME_BEFORE_SIG_KILL =
"mapreduce.tasktracker.tasks.sleeptimebeforesigkill";
public static final String TT_DNS_INTERFACE =
"mapreduce.tasktracker.dns.interface";
public static final String TT_DNS_NAMESERVER =
"mapreduce.tasktracker.dns.nameserver";
public static final String TT_MAX_TASK_COMPLETION_EVENTS_TO_POLL =
"mapreduce.tasktracker.events.batchsize";
public static final String TT_INDEX_CACHE =
"mapreduce.tasktracker.indexcache.mb";
public static final String TT_INSTRUMENTATION =
"mapreduce.tasktracker.instrumentation";
public static final String TT_MAP_SLOTS =
"mapreduce.tasktracker.map.tasks.maximum";
/**
* @deprecated Use {@link #TT_RESOURCE_CALCULATOR_PLUGIN} instead
*/
@Deprecated
public static final String TT_MEMORY_CALCULATOR_PLUGIN =
"mapreduce.tasktracker.memorycalculatorplugin";
public static final String TT_RESOURCE_CALCULATOR_PLUGIN =
"mapreduce.tasktracker.resourcecalculatorplugin";
public static final String TT_REDUCE_SLOTS =
"mapreduce.tasktracker.reduce.tasks.maximum";
public static final String TT_LOCAL_CACHE_SIZE =
"mapreduce.tasktracker.cache.local.size";
public static final String TT_LOCAL_CACHE_SUBDIRS_LIMIT =
"mapreduce.tasktracker.cache.local.numberdirectories";
public static final String TT_OUTOFBAND_HEARBEAT =
"mapreduce.tasktracker.outofband.heartbeat";
public static final String TT_RESERVED_PHYSCIALMEMORY_MB =
"mapreduce.tasktracker.reserved.physicalmemory.mb";
public static final String TT_USER_NAME = "mapreduce.tasktracker.kerberos.principal";
public static final String TT_KEYTAB_FILE =
"mapreduce.tasktracker.keytab.file";
public static final String TT_GROUP =
"mapreduce.tasktracker.group";
public static final String TT_USERLOGCLEANUP_SLEEPTIME =
"mapreduce.tasktracker.userlogcleanup.sleeptime";
public static final String TT_DISTRIBUTED_CACHE_CHECK_PERIOD =
"mapreduce.tasktracker.distributedcache.checkperiod";
/**
* Percentage of the local distributed cache that should be kept in between
* garbage collection.
*/
public static final String TT_LOCAL_CACHE_KEEP_AROUND_PCT =
"mapreduce.tasktracker.cache.local.keep.pct";
}

41
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
View File

@ -80,55 +80,14 @@ private static void addDeprecatedKeys() {
JTConfig.JT_TASKCACHE_LEVELS),
new DeprecationDelta("mapred.job.tracker.retire.jobs",
JTConfig.JT_RETIREJOBS),
new DeprecationDelta("mapred.healthChecker.interval",
TTConfig.TT_HEALTH_CHECKER_INTERVAL),
new DeprecationDelta("mapred.healthChecker.script.args",
TTConfig.TT_HEALTH_CHECKER_SCRIPT_ARGS),
new DeprecationDelta("mapred.healthChecker.script.path",
TTConfig.TT_HEALTH_CHECKER_SCRIPT_PATH),
new DeprecationDelta("mapred.healthChecker.script.timeout",
TTConfig.TT_HEALTH_CHECKER_SCRIPT_TIMEOUT),
new DeprecationDelta("mapred.local.dir.minspacekill",
TTConfig.TT_LOCAL_DIR_MINSPACE_KILL),
new DeprecationDelta("mapred.local.dir.minspacestart",
TTConfig.TT_LOCAL_DIR_MINSPACE_START),
new DeprecationDelta("mapred.task.tracker.http.address",
TTConfig.TT_HTTP_ADDRESS),
new DeprecationDelta("mapred.task.tracker.report.address",
TTConfig.TT_REPORT_ADDRESS),
new DeprecationDelta("mapred.task.tracker.task-controller",
TTConfig.TT_TASK_CONTROLLER),
new DeprecationDelta("mapred.tasktracker.dns.interface",
TTConfig.TT_DNS_INTERFACE),
new DeprecationDelta("mapred.tasktracker.dns.nameserver",
TTConfig.TT_DNS_NAMESERVER),
new DeprecationDelta("mapred.tasktracker.events.batchsize",
TTConfig.TT_MAX_TASK_COMPLETION_EVENTS_TO_POLL),
new DeprecationDelta("mapred.tasktracker.indexcache.mb",
TTConfig.TT_INDEX_CACHE),
new DeprecationDelta("mapred.tasktracker.instrumentation",
TTConfig.TT_INSTRUMENTATION),
new DeprecationDelta("mapred.tasktracker.map.tasks.maximum",
TTConfig.TT_MAP_SLOTS),
new DeprecationDelta("mapred.tasktracker.memory_calculator_plugin",
TTConfig.TT_RESOURCE_CALCULATOR_PLUGIN),
new DeprecationDelta("mapred.tasktracker.memorycalculatorplugin",
TTConfig.TT_RESOURCE_CALCULATOR_PLUGIN),
new DeprecationDelta("mapred.tasktracker.reduce.tasks.maximum",
TTConfig.TT_REDUCE_SLOTS),
new DeprecationDelta(
"mapred.tasktracker.tasks.sleeptime-before-sigkill",
TTConfig.TT_SLEEP_TIME_BEFORE_SIG_KILL),
new DeprecationDelta("slave.host.name",
TTConfig.TT_HOST_NAME),
new DeprecationDelta("tasktracker.http.threads",
TTConfig.TT_HTTP_THREADS),
new DeprecationDelta("hadoop.net.static.resolutions",
TTConfig.TT_STATIC_RESOLUTIONS),
new DeprecationDelta("local.cache.size",
TTConfig.TT_LOCAL_CACHE_SIZE),
new DeprecationDelta("tasktracker.contention.tracking",
TTConfig.TT_CONTENTION_TRACKING),
new DeprecationDelta("yarn.app.mapreduce.yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts",
MRJobConfig.MR_CLIENT_TO_AM_IPC_MAX_RETRIES_ON_TIMEOUTS),
new DeprecationDelta("job.end.notification.url",

13
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
View File

@ -6,7 +6,6 @@
<property><!--Loaded from job.xml--><name>yarn.nodemanager.container-manager.thread-count</name><value>20</value></property>
<property><!--Loaded from job.xml--><name>mapred.queue.default.acl-administer-jobs</name><value>*</value></property>
<property><!--Loaded from job.xml--><name>dfs.image.transfer.bandwidthPerSec</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.healthchecker.interval</name><value>60000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.staging.root.dir</name><value>${hadoop.tmp.dir}/mapred/staging</value></property>
<property><!--Loaded from job.xml--><name>dfs.block.access.token.lifetime</name><value>600</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.am.max-retries</name><value>2</value></property>
@ -56,19 +55,16 @@
<property><!--Loaded from job.xml--><name>dfs.namenode.replication.min</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.java.opts</name><value>-Xmx500m</value></property>
<property><!--Loaded from job.xml--><name>s3native.bytes-per-checksum</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.tasks.sleeptimebeforesigkill</name><value>5000</value></property>
<property><!--Loaded from job.xml--><name>tfile.fs.output.buffer.size</name><value>262144</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.local-dirs</name><value>/home/user/local-dir/</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.persist.jobstatus.active</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>fs.AbstractFileSystem.hdfs.impl</name><value>org.apache.hadoop.fs.Hdfs</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.safemode.min.datanodes</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.local.dir.minspacestart</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.client.https.need-auth</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>fs.har.impl.disable.cache</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>dfs.client.https.keystore.resource</name><value>ssl-client.xml</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.max.objects</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.safemode.threshold-pct</name><value>0.999f</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.local.dir.minspacekill</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.reduce.slowstart.completedmaps</name><value>0.05</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.end-notification.retry.attempts</name><value>5</value></property>
@ -76,7 +72,6 @@
<property><!--Loaded from job.xml--><name>fs.s3n.impl</name><value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.memory.mb</name><value>512</value></property>
<property><!--Loaded from Unknown--><name>mapreduce.job.user.name</name><value>user</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.outofband.heartbeat</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>io.native.lib.available</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>dfs.client-write-packet-size</name><value>65536</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.client.progressmonitor.pollinterval</name><value>1000</value></property>
@ -90,7 +85,6 @@
<property><!--Loaded from job.xml--><name>mapreduce.cluster.local.dir</name><value>${hadoop.tmp.dir}/mapred/local</value></property>
<property><!--Loaded from Unknown--><name>mapreduce.job.application.attempt.id</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>dfs.permissions.enabled</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.taskcontroller</name><value>org.apache.hadoop.mapred.DefaultTaskController</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.parallelcopies</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.env-whitelist</name><value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.maxtaskfailures.per.tracker</name><value>4</value></property>
@ -138,7 +132,6 @@
<property><!--Loaded from job.xml--><name>mapreduce.map.output.key.class</name><value>org.apache.hadoop.io.IntWritable</value></property>
<property><!--Loaded from mapred-default.xml--><name>mapreduce.job.end-notification.max.retry.interval</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>ftp.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.http.threads</name><value>40</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.java.opts</name><value>-Xmx500m</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.data.dir</name><value>file:///home/user/hadoop-user/dfs/data</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.replication.interval</name><value>3</value></property>
@ -194,7 +187,6 @@
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.speculative</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.client.output.filter</name><value>FAILED</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.report.address</name><value>127.0.0.1:0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.userlog.limit.kb</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.map.tasks.maximum</name><value>2</value></property>
<property><!--Loaded from job.xml--><name>hadoop.http.authentication.simple.anonymous.allowed</name><value>true</value></property>
@ -204,7 +196,6 @@
<property><!--Loaded from job.xml--><name>fs.automatic.close</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>fs.kfs.impl</name><value>org.apache.hadoop.fs.kfs.KosmosFileSystem</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.submithostaddress</name><value>127.0.0.1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.healthchecker.script.timeout</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.directoryscan.interval</name><value>21600</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.address</name><value>0.0.0.0:8040</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.log-aggregation-enable</name><value>false</value></property>
@ -230,8 +221,6 @@
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.container.liveness-monitor.interval-ms</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>yarn.am.liveness-monitor.expiry-interval-ms</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.profile</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.instrumentation</name><value>org.apache.hadoop.mapred.TaskTrackerMetricsInst</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.http.address</name><value>0.0.0.0:50060</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobhistory.webapp.address</name><value>0.0.0.0:19888</value></property>
<property><!--Loaded from Unknown--><name>rpc.engine.org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$BlockingInterface</name><value>org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine</value></property>
<property><!--Loaded from job.xml--><name>yarn.ipc.rpc.class</name><value>org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC</value></property>
@ -271,7 +260,6 @@
<property><!--Loaded from job.xml--><name>mapreduce.admin.user.env</name><value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.checkpoint.edits.dir</name><value>${dfs.namenode.checkpoint.dir}</value></property>
<property><!--Loaded from job.xml--><name>hadoop.common.configuration.version</name><value>0.23.0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.dns.interface</name><value>default</value></property>
<property><!--Loaded from job.xml--><name>io.serializations</name><value>org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.aux-service.mapreduce.shuffle.class</name><value>org.apache.hadoop.mapred.ShuffleHandler</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name><value>org.apache.hadoop.mapred.ShuffleHandler</value></property>
@ -352,7 +340,6 @@
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.admin.client.thread-count</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>fs.AbstractFileSystem.viewfs.impl</name><value>org.apache.hadoop.fs.viewfs.ViewFs</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.resource-tracker.client.thread-count</name><value>50</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.dns.nameserver</name><value>default</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.output.compress</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.counters.limit</name><value>120</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.ipc.address</name><value>0.0.0.0:9867</value></property>

26
hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
View File

@ -4532,7 +4532,6 @@
"yarn.nodemanager.container-manager.thread-count" : "20",
"mapred.queue.default.acl-administer-jobs" : "*",
"dfs.image.transfer.bandwidthPerSec" : "0",
"mapreduce.tasktracker.healthchecker.interval" : "60000",
"mapreduce.jobtracker.staging.root.dir" : "/user",
"yarn.resourcemanager.recovery.enabled" : "false",
"yarn.resourcemanager.am.max-retries" : "1",
@ -4606,13 +4605,11 @@
"mapreduce.map.java.opts" : " -Xmx825955249",
"yarn.scheduler.fair.allocation.file" : "/etc/yarn/fair-scheduler.xml",
"s3native.bytes-per-checksum" : "512",
"mapreduce.tasktracker.tasks.sleeptimebeforesigkill" : "5000",
"tfile.fs.output.buffer.size" : "262144",
"yarn.nodemanager.local-dirs" : "${hadoop.tmp.dir}/nm-local-dir",
"mapreduce.jobtracker.persist.jobstatus.active" : "false",
"fs.AbstractFileSystem.hdfs.impl" : "org.apache.hadoop.fs.Hdfs",
"mapreduce.job.map.output.collector.class" : "org.apache.hadoop.mapred.MapTask$MapOutputBuffer",
"mapreduce.tasktracker.local.dir.minspacestart" : "0",
"dfs.namenode.safemode.min.datanodes" : "0",
"hadoop.security.uid.cache.secs" : "14400",
"dfs.client.https.need-auth" : "false",
@ -4621,7 +4618,6 @@
"dfs.namenode.max.objects" : "0",
"hadoop.ssl.client.conf" : "ssl-client.xml",
"dfs.namenode.safemode.threshold-pct" : "0.999f",
"mapreduce.tasktracker.local.dir.minspacekill" : "0",
"dfs.blocksize" : "134217728",
"yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler",
"mapreduce.job.reduce.slowstart.completedmaps" : "0.8",
@ -4629,7 +4625,6 @@
"mapreduce.job.inputformat.class" : "org.apache.hadoop.examples.terasort.TeraGen$RangeInputFormat",
"mapreduce.map.memory.mb" : "1024",
"mapreduce.job.user.name" : "jenkins",
"mapreduce.tasktracker.outofband.heartbeat" : "false",
"io.native.lib.available" : "true",
"dfs.client-write-packet-size" : "65536",
"mapreduce.client.progressmonitor.pollinterval" : "1000",
@ -4644,7 +4639,6 @@
"yarn.scheduler.fair.user-as-default-queue" : "true",
"mapreduce.job.application.attempt.id" : "1",
"dfs.permissions.enabled" : "true",
"mapreduce.tasktracker.taskcontroller" : "org.apache.hadoop.mapred.DefaultTaskController",
"yarn.scheduler.fair.preemption" : "true",
"mapreduce.reduce.shuffle.parallelcopies" : "5",
"yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
@ -4700,7 +4694,6 @@
"mapreduce.reduce.shuffle.retry-delay.max.ms" : "60000",
"mapreduce.job.end-notification.max.retry.interval" : "5",
"ftp.blocksize" : "67108864",
"mapreduce.tasktracker.http.threads" : "80",
"mapreduce.reduce.java.opts" : " -Xmx825955249",
"dfs.datanode.data.dir" : "file://${hadoop.tmp.dir}/dfs/data",
"ha.failover-controller.cli-check.rpc-timeout.ms" : "20000",
@ -4768,7 +4761,6 @@
"mapreduce.reduce.speculative" : "false",
"mapreduce.client.output.filter" : "FAILED",
"mapreduce.ifile.readahead.bytes" : "4194304",
"mapreduce.tasktracker.report.address" : "127.0.0.1:0",
"mapreduce.task.userlog.limit.kb" : "0",
"mapreduce.tasktracker.map.tasks.maximum" : "2",
"hadoop.http.authentication.simple.anonymous.allowed" : "true",
@ -4779,7 +4771,6 @@
"dfs.namenode.handler.count" : "10",
"fs.automatic.close" : "false",
"mapreduce.job.submithostaddress" : "10.20.206.115",
"mapreduce.tasktracker.healthchecker.script.timeout" : "600000",
"dfs.datanode.directoryscan.interval" : "21600",
"yarn.resourcemanager.address" : "a2115.smile.com:8032",
"yarn.nodemanager.health-checker.interval-ms" : "600000",
@ -4809,8 +4800,6 @@
"yarn.resourcemanager.container.liveness-monitor.interval-ms" : "600000",
"yarn.am.liveness-monitor.expiry-interval-ms" : "600000",
"mapreduce.task.profile" : "false",
"mapreduce.tasktracker.http.address" : "0.0.0.0:50060",
"mapreduce.tasktracker.instrumentation" : "org.apache.hadoop.mapred.TaskTrackerMetricsInst",
"mapreduce.jobhistory.webapp.address" : "a2115.smile.com:19888",
"ha.failover-controller.graceful-fence.rpc-timeout.ms" : "5000",
"yarn.ipc.rpc.class" : "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
@ -4857,7 +4846,6 @@
"fs.permissions.umask-mode" : "022",
"dfs.client.domain.socket.data.traffic" : "false",
"hadoop.common.configuration.version" : "0.23.0",
"mapreduce.tasktracker.dns.interface" : "default",
"mapreduce.output.fileoutputformat.compress.type" : "BLOCK",
"mapreduce.ifile.readahead" : "true",
"hadoop.security.group.mapping.ldap.ssl" : "false",
@ -4945,7 +4933,6 @@
"dfs.namenode.checkpoint.period" : "3600",
"fs.AbstractFileSystem.viewfs.impl" : "org.apache.hadoop.fs.viewfs.ViewFs",
"yarn.resourcemanager.resource-tracker.client.thread-count" : "50",
"mapreduce.tasktracker.dns.nameserver" : "default",
"mapreduce.map.output.compress" : "true",
"dfs.datanode.ipc.address" : "0.0.0.0:9867",
"hadoop.ssl.require.client.cert" : "false",
@ -9613,7 +9600,6 @@
"yarn.nodemanager.container-manager.thread-count" : "20",
"mapred.queue.default.acl-administer-jobs" : "*",
"dfs.image.transfer.bandwidthPerSec" : "0",
"mapreduce.tasktracker.healthchecker.interval" : "60000",
"mapreduce.jobtracker.staging.root.dir" : "/user",
"yarn.resourcemanager.recovery.enabled" : "false",
"yarn.resourcemanager.am.max-retries" : "1",
@ -9687,13 +9673,11 @@
"mapreduce.map.java.opts" : " -Xmx825955249",
"yarn.scheduler.fair.allocation.file" : "/etc/yarn/fair-scheduler.xml",
"s3native.bytes-per-checksum" : "512",
"mapreduce.tasktracker.tasks.sleeptimebeforesigkill" : "5000",
"tfile.fs.output.buffer.size" : "262144",
"yarn.nodemanager.local-dirs" : "${hadoop.tmp.dir}/nm-local-dir",
"mapreduce.jobtracker.persist.jobstatus.active" : "false",
"fs.AbstractFileSystem.hdfs.impl" : "org.apache.hadoop.fs.Hdfs",
"mapreduce.job.map.output.collector.class" : "org.apache.hadoop.mapred.MapTask$MapOutputBuffer",
"mapreduce.tasktracker.local.dir.minspacestart" : "0",
"dfs.namenode.safemode.min.datanodes" : "0",
"hadoop.security.uid.cache.secs" : "14400",
"dfs.client.https.need-auth" : "false",
@ -9702,7 +9686,6 @@
"dfs.namenode.max.objects" : "0",
"hadoop.ssl.client.conf" : "ssl-client.xml",
"dfs.namenode.safemode.threshold-pct" : "0.999f",
"mapreduce.tasktracker.local.dir.minspacekill" : "0",
"dfs.blocksize" : "134217728",
"yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler",
"mapreduce.job.reduce.slowstart.completedmaps" : "0.8",
@ -9710,7 +9693,6 @@
"mapreduce.job.inputformat.class" : "org.apache.hadoop.examples.terasort.TeraGen$RangeInputFormat",
"mapreduce.map.memory.mb" : "1024",
"mapreduce.job.user.name" : "jenkins",
"mapreduce.tasktracker.outofband.heartbeat" : "false",
"io.native.lib.available" : "true",
"dfs.client-write-packet-size" : "65536",
"mapreduce.client.progressmonitor.pollinterval" : "1000",
@ -9725,7 +9707,6 @@
"yarn.scheduler.fair.user-as-default-queue" : "true",
"mapreduce.job.application.attempt.id" : "1",
"dfs.permissions.enabled" : "true",
"mapreduce.tasktracker.taskcontroller" : "org.apache.hadoop.mapred.DefaultTaskController",
"yarn.scheduler.fair.preemption" : "true",
"mapreduce.reduce.shuffle.parallelcopies" : "5",
"yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
@ -9781,7 +9762,6 @@
"mapreduce.reduce.shuffle.retry-delay.max.ms" : "60000",
"mapreduce.job.end-notification.max.retry.interval" : "5",
"ftp.blocksize" : "67108864",
"mapreduce.tasktracker.http.threads" : "80",
"mapreduce.reduce.java.opts" : " -Xmx825955249",
"dfs.datanode.data.dir" : "file://${hadoop.tmp.dir}/dfs/data",
"ha.failover-controller.cli-check.rpc-timeout.ms" : "20000",
@ -9849,7 +9829,6 @@
"mapreduce.reduce.speculative" : "false",
"mapreduce.client.output.filter" : "FAILED",
"mapreduce.ifile.readahead.bytes" : "4194304",
"mapreduce.tasktracker.report.address" : "127.0.0.1:0",
"mapreduce.task.userlog.limit.kb" : "0",
"mapreduce.tasktracker.map.tasks.maximum" : "2",
"hadoop.http.authentication.simple.anonymous.allowed" : "true",
@ -9860,7 +9839,6 @@
"dfs.namenode.handler.count" : "10",
"fs.automatic.close" : "false",
"mapreduce.job.submithostaddress" : "10.20.206.115",
"mapreduce.tasktracker.healthchecker.script.timeout" : "600000",
"dfs.datanode.directoryscan.interval" : "21600",
"yarn.resourcemanager.address" : "a2115.smile.com:8032",
"yarn.nodemanager.health-checker.interval-ms" : "600000",
@ -9890,8 +9868,6 @@
"yarn.resourcemanager.container.liveness-monitor.interval-ms" : "600000",
"yarn.am.liveness-monitor.expiry-interval-ms" : "600000",
"mapreduce.task.profile" : "false",
"mapreduce.tasktracker.http.address" : "0.0.0.0:50060",
"mapreduce.tasktracker.instrumentation" : "org.apache.hadoop.mapred.TaskTrackerMetricsInst",
"mapreduce.jobhistory.webapp.address" : "a2115.smile.com:19888",
"ha.failover-controller.graceful-fence.rpc-timeout.ms" : "5000",
"yarn.ipc.rpc.class" : "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
@ -9938,7 +9914,6 @@
"fs.permissions.umask-mode" : "022",
"dfs.client.domain.socket.data.traffic" : "false",
"hadoop.common.configuration.version" : "0.23.0",
"mapreduce.tasktracker.dns.interface" : "default",
"mapreduce.output.fileoutputformat.compress.type" : "BLOCK",
"mapreduce.ifile.readahead" : "true",
"hadoop.security.group.mapping.ldap.ssl" : "false",
@ -10026,7 +10001,6 @@
"dfs.namenode.checkpoint.period" : "3600",
"fs.AbstractFileSystem.viewfs.impl" : "org.apache.hadoop.fs.viewfs.ViewFs",
"yarn.resourcemanager.resource-tracker.client.thread-count" : "50",
"mapreduce.tasktracker.dns.nameserver" : "default",
"mapreduce.map.output.compress" : "true",
"dfs.datanode.ipc.address" : "0.0.0.0:9867",
"hadoop.ssl.require.client.cert" : "false",