big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5517. Fixed MapReduce ApplicationMaster to not validate reduce side resource configuration for deciding uber-mode on map-only jobs. Contributed by Siqi Li.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1608595 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-07-07 20:14:01 +00:00
parent 286ae1edd0
commit 4b2ded8202
3 changed files with 31 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -297,6 +297,10 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5868. Fixed an issue with TestPipeApplication that was causing the
nightly builds to fail. (Akira Ajisaka via vinodkv)
MAPREDUCE-5517. Fixed MapReduce ApplicationMaster to not validate reduce side
resource configuration for deciding uber-mode on map-only jobs. (Siqi Li via
vinodkv)
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES

33
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
View File

@ -1218,22 +1218,25 @@ private void makeUberDecision(long dataInputLength) {
boolean smallNumReduceTasks = (numReduceTasks <= sysMaxReduces);
boolean smallInput = (dataInputLength <= sysMaxBytes);
// ignoring overhead due to UberAM and statics as negligible here:
long requiredMapMB = conf.getLong(MRJobConfig.MAP_MEMORY_MB, 0);
long requiredReduceMB = conf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 0);
long requiredMB = Math.max(requiredMapMB, requiredReduceMB);
int requiredMapCores = conf.getInt(
MRJobConfig.MAP_CPU_VCORES,
MRJobConfig.DEFAULT_MAP_CPU_VCORES);
int requiredReduceCores = conf.getInt(
MRJobConfig.REDUCE_CPU_VCORES,
MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
int requiredCores = Math.max(requiredMapCores, requiredReduceCores);
if (numReduceTasks == 0) {
requiredMB = requiredMapMB;
requiredCores = requiredMapCores;
}
boolean smallMemory =
( (Math.max(conf.getLong(MRJobConfig.MAP_MEMORY_MB, 0),
conf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 0))
<= sysMemSizeForUberSlot)
|| (sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT));
boolean smallCpu =
(
Math.max(
conf.getInt(
MRJobConfig.MAP_CPU_VCORES,
MRJobConfig.DEFAULT_MAP_CPU_VCORES),
conf.getInt(
MRJobConfig.REDUCE_CPU_VCORES,
MRJobConfig.DEFAULT_REDUCE_CPU_VCORES))
<= sysCPUSizeForUberSlot
);
(requiredMB <= sysMemSizeForUberSlot)
|| (sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT);
boolean smallCpu = requiredCores <= sysCPUSizeForUberSlot;
boolean notChainJob = !isChainJob(conf);
// User has overall veto power over uberization, or user can modify

9
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
View File

@ -657,6 +657,15 @@ public void testUberDecision() throws Exception {
conf.setInt(MRJobConfig.JOB_UBERTASK_MAXMAPS, 1);
isUber = testUberDecision(conf);
Assert.assertFalse(isUber);
// enable uber mode of 0 reducer no matter how much memory assigned to reducer
conf = new Configuration();
conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true);
conf.setInt(MRJobConfig.NUM_REDUCES, 0);
conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 2048);
conf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 10);
isUber = testUberDecision(conf);
Assert.assertTrue(isUber);
}
private boolean testUberDecision(Configuration conf) {