MAPREDUCE-3954. Added new envs to separate heap size for different daemons started via bin scripts. Contributed by Robert Joseph Evans.

.--This line, and those below, will be ignored--

M    hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
M    hadoop-mapreduce-project/hadoop-yarn/bin/yarn
M    hadoop-mapreduce-project/CHANGES.txt
M    hadoop-mapreduce-project/bin/mapred


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1297174 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2012-03-05 19:03:56 +00:00
parent b73738d21e
commit 7c1e857176
4 changed files with 42 additions and 9 deletions

View File

@ -159,6 +159,9 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3497. Added docs for YARN CLI. (tgraves via acmurthy) MAPREDUCE-3497. Added docs for YARN CLI. (tgraves via acmurthy)
MAPREDUCE-3954. Added new envs to separate heap size for different daemons
started via bin scripts. (Robert Joseph Evans via vinodkv)
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-3901. Modified JobHistory records in YARN to lazily load job and MAPREDUCE-3901. Modified JobHistory records in YARN to lazily load job and

View File

@ -51,8 +51,6 @@ fi
COMMAND=$1 COMMAND=$1
shift shift
HADOOP_JOB_HISTORYSERVER_OPTS="-Dmapred.jobsummary.logger=${HADOOP_JHS_LOGGER:-INFO,console}"
if [ "$COMMAND" = "job" ] ; then if [ "$COMMAND" = "job" ] ; then
CLASS=org.apache.hadoop.mapred.JobClient CLASS=org.apache.hadoop.mapred.JobClient
elif [ "$COMMAND" = "queue" ] ; then elif [ "$COMMAND" = "queue" ] ; then
@ -70,7 +68,10 @@ elif [ "$COMMAND" = "groups" ] ; then
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
elif [ "$COMMAND" = "historyserver" ] ; then elif [ "$COMMAND" = "historyserver" ] ; then
CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOB_HISTORYSERVER_OPTS" HADOOP_OPTS="$HADOOP_OPTS -Dmapred.jobsummary.logger=${HADOOP_JHS_LOGGER:-INFO,console} $HADOOP_JOB_HISTORYSERVER_OPTS"
if [ "$HADOOP_JOB_HISTORYSERVER_HEAPSIZE" != "" ]; then
JAVA_HEAP_MAX="-Xmx""$HADOOP_JOB_HISTORYSERVER_HEAPSIZE""m"
fi
elif [ "$COMMAND" = "mradmin" ] \ elif [ "$COMMAND" = "mradmin" ] \
|| [ "$COMMAND" = "jobtracker" ] \ || [ "$COMMAND" = "jobtracker" ] \
|| [ "$COMMAND" = "tasktracker" ] ; then || [ "$COMMAND" = "tasktracker" ] ; then

View File

@ -27,11 +27,16 @@
# YARN_HEAPSIZE The maximum amount of heap to use, in MB. # YARN_HEAPSIZE The maximum amount of heap to use, in MB.
# Default is 1000. # Default is 1000.
# #
# YARN_{COMMAND}_HEAPSIZE overrides YARN_HEAPSIZE for a given command
# eg YARN_NODEMANAGER_HEAPSIZE sets the heap
# size for the NodeManager. If you set the
# heap size in YARN_{COMMAND}_OPTS or YARN_OPTS
# they take precedence.
#
# YARN_OPTS Extra Java runtime options. # YARN_OPTS Extra Java runtime options.
# #
# YARN_NAMENODE_OPTS These options are added to YARN_OPTS
# YARN_CLIENT_OPTS when the respective command is run. # YARN_CLIENT_OPTS when the respective command is run.
# YARN_{COMMAND}_OPTS etc YARN_JT_OPTS applies to JobTracker # YARN_{COMMAND}_OPTS etc YARN_NODEMANAGER_OPTS applies to NodeManager
# for e.g. YARN_CLIENT_OPTS applies to # for e.g. YARN_CLIENT_OPTS applies to
# more than one command (fs, dfs, fsck, # more than one command (fs, dfs, fsck,
# dfsadmin etc) # dfsadmin etc)
@ -170,6 +175,9 @@ elif [ "$COMMAND" = "resourcemanager" ] ; then
CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS" YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS"
if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then
JAVA_HEAP_MAX="-Xmx""$YARN_RESOURCEMANAGER_HEAPSIZE""m"
fi
elif [ "$COMMAND" = "nodemanager" ] ; then elif [ "$COMMAND" = "nodemanager" ] ; then
CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties
CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager' CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
@ -178,9 +186,15 @@ elif [ "$COMMAND" = "nodemanager" ] ; then
else else
YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS" YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS"
fi fi
if [ "$YARN_NODEMANAGER_HEAPSIZE" != "" ]; then
JAVA_HEAP_MAX="-Xmx""$YARN_NODEMANAGER_HEAPSIZE""m"
fi
elif [ "$COMMAND" = "proxyserver" ] ; then elif [ "$COMMAND" = "proxyserver" ] ; then
CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer' CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
YARN_OPTS="$YARN_OPTS $YARN_PROXYSERVER_OPTS" YARN_OPTS="$YARN_OPTS $YARN_PROXYSERVER_OPTS"
if [ "$YARN_PROXYSERVER_HEAPSIZE" != "" ]; then
JAVA_HEAP_MAX="-Xmx""$YARN_PROXYSERVER_HEAPSIZE""m"
fi
elif [ "$COMMAND" = "version" ] ; then elif [ "$COMMAND" = "version" ] ; then
CLASS=org.apache.hadoop.util.VersionInfo CLASS=org.apache.hadoop.util.VersionInfo
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"

View File

@ -103,6 +103,9 @@ Hadoop MapReduce Next Generation - Cluster Setup
*--------------------------------------+--------------------------------------+ *--------------------------------------+--------------------------------------+
| WebAppProxy | YARN_PROXYSERVER_OPTS | | WebAppProxy | YARN_PROXYSERVER_OPTS |
*--------------------------------------+--------------------------------------+ *--------------------------------------+--------------------------------------+
| Map Reduce Job History Server | HADOOP_JOB_HISTORYSERVER_OPTS |
*--------------------------------------+--------------------------------------+
For example, To configure Namenode to use parallelGC, the following For example, To configure Namenode to use parallelGC, the following
statement should be added in hadoop-env.sh : statement should be added in hadoop-env.sh :
@ -118,9 +121,21 @@ Hadoop MapReduce Next Generation - Cluster Setup
don't exist. don't exist.
* <<<HADOOP_HEAPSIZE>>> / <<<YARN_HEAPSIZE>>> - The maximum amount of * <<<HADOOP_HEAPSIZE>>> / <<<YARN_HEAPSIZE>>> - The maximum amount of
heapsize to use, in MB e.g. 1000MB. This is used to configure the heap heapsize to use, in MB e.g. if the varibale is set to 1000 the heap
size for the daemon. By default, the value is 1000MB. will be set to 1000MB. This is used to configure the heap
size for the daemon. By default, the value is 1000. If you want to
configure the values separately for each deamon you can use.
*--------------------------------------+--------------------------------------+
|| Daemon || Environment Variable |
*--------------------------------------+--------------------------------------+
| ResourceManager | YARN_RESOURCEMANAGER_HEAPSIZE |
*--------------------------------------+--------------------------------------+
| NodeManager | YARN_NODEMANAGER_HEAPSIZE |
*--------------------------------------+--------------------------------------+
| WebAppProxy | YARN_PROXYSERVER_HEAPSIZE |
*--------------------------------------+--------------------------------------+
| Map Reduce Job History Server | HADOOP_JOB_HISTORYSERVER_HEAPSIZE |
*--------------------------------------+--------------------------------------+
* {Configuring the Hadoop Daemons in Non-Secure Mode} * {Configuring the Hadoop Daemons in Non-Secure Mode}