big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-7728. Enable task memory management to be configurable in hadoop config setup script. (ramya)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1190035 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Ramya Sunil 2011-10-27 21:11:16 +00:00
parent b9f6890db9
commit 072ef952c3
3 changed files with 79 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -55,6 +55,9 @@ Trunk (unreleased changes)
HADOOP-7758. Make GlobFilter class public. (tucu)
HADOOP-7728. Enable task memory management to be configurable in hadoop
config setup script. (ramya)
BUGS
HADOOP-7606. Upgrade Jackson to version 1.7.1 to match the version required

46
hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
View File

@ -55,6 +55,16 @@ usage: $0 <parameters>
--dfs-support-append=false|true Enable append
--hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts' Setup proxy users for hadoop
--hbase-user=hbase User which hbase is running as. Defaults to hbase
--mapreduce-cluster-mapmemory-mb=memory Virtual memory of a map slot for the MR framework. Defaults to -1
--mapreduce-cluster-reducememory-mb=memory Virtual memory, of a reduce slot for the MR framework. Defaults to -1
--mapreduce-jobtracker-maxmapmemory-mb=memory Maximum virtual memory of a single map task. Defaults to -1
This value should be set to (mapreduce.cluster.mapmemory.mb * mapreduce.tasktracker.map.tasks.maximum)
--mapreduce-jobtracker-maxreducememory-mb=memory Maximum virtual memory of a single reduce task. Defaults to -1
This value should be set to (mapreduce.cluster.reducememory.mb * mapreduce.tasktracker.reduce.tasks.maximum)
--mapreduce-map-memory-mb=memory Virtual memory of a single map slot for a job. Defaults to -1
This value should be <= mapred.cluster.max.map.memory.mb
--mapreduce-reduce-memory-mb=memory Virtual memory, of a single reduce slot for a job. Defaults to -1
This value should be <= mapred.cluster.max.reduce.memory.mb
"
exit 1
}
@ -199,6 +209,12 @@ OPTS=$(getopt \
-l 'hadoop-proxy-users:' \
-l 'dfs-support-append:' \
-l 'hbase-user:' \
-l 'mapreduce-cluster-mapmemory-mb:' \
-l 'mapreduce-cluster-reducememory-mb:' \
-l 'mapreduce-jobtracker-maxmapmemory-mb:' \
-l 'mapreduce-jobtracker-maxreducememory-mb:' \
-l 'mapreduce-map-memory-mb:' \
-l 'mapreduce-reduce-memory-mb:' \
-o 'h' \
-- "$@")
@ -334,6 +350,30 @@ while true ; do
HBASE_USER=$2; shift 2
AUTOMATED=1
;;
--mapreduce-cluster-mapmemory-mb)
MAPREDUCE_CLUSTER_MAPMEMORY_MB=$2; shift 2
AUTOMATED=1
;;
--mapreduce-cluster-reducememory-mb)
MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=$2; shift 2
AUTOMATED=1
;;
--mapreduce-jobtracker-maxmapmemory-mb)
MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=$2; shift 2
AUTOMATED=1
;;
--mapreduce-jobtracker-maxreducememory-mb)
MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=$2; shift 2
AUTOMATED=1
;;
--mapreduce-map-memory-mb)
MAPREDUCE_MAP_MEMORY_MB=$2; shift 2
AUTOMATED=1
;;
--mapreduce-reduce-memory-mb)
MAPREDUCE_REDUCE_MEMORY_MB=$2; shift 2
AUTOMATED=1
;;
--)
shift ; break
;;
@ -365,6 +405,12 @@ HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
HBASE_USER=${HBASE_USER:-hbase}
MAPREDUCE_CLUSTER_MAPMEMORY_MB=${MAPREDUCE_CLUSTER_MAPMEMORY_MB:--1}
MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB:--1}
MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB:--1}
MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB:--1}
MAPREDUCE_MAP_MEMORY_MB=${MAPREDUCE_MAP_MEMORY_MB:--1}
MAPREDUCE_REDUCE_MEMORY_MB=${MAPREDUCE_REDUCE_MEMORY_MB:--1}
KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}

30
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
View File

@ -280,4 +280,34 @@
<name>mapred.jobtracker.retirejob.interval</name>
<value>0</value>
</property>
<property>
<name>mapreduce.cluster.mapmemory.mb</name>
<value>${MAPREDUCE_CLUSTER_MAPMEMORY_MB}</value>
</property>
<property>
<name>mapreduce.cluster.reducememory.mb</name>
<value>${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB}</value>
</property>
<property>
<name>mapreduce.jobtracker.maxmapmemory.mb</name>
<value>${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB}</value>
</property>
<property>
<name>mapreduce.jobtracker.maxreducememory.mb</name>
<value>${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB}</value>
</property>
<property>
<name>mapreduce.map.memory.mb</name>
<value>${MAPREDUCE_MAP_MEMORY_MB}</value>
</property>
<property>
<name>mapreduce.reduce.memory.mb</name>
<value>${MAPREDUCE_REDUCE_MEMORY_MB}</value>
</property>
</configuration>