big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6223. TestJobConf#testNegativeValueForTaskVmem failures. (Varun Saxena via kasha)

Browse Source
This commit is contained in:
Karthik Kambatla 2015-02-26 14:24:19 -08:00
parent c6d5b37a1c
commit 1047c883ba
3 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -390,6 +390,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-2815. JavaDoc does not generate correctly for MAPREDUCE-2815. JavaDoc does not generate correctly for
MultithreadedMapRunner. (Chris Palmer via aajisaka) MultithreadedMapRunner. (Chris Palmer via aajisaka)
MAPREDUCE-6223. TestJobConf#testNegativeValueForTaskVmem failures.
(Varun Saxena via kasha)
Release 2.6.1 - UNRELEASED Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
View File

@ -1826,8 +1826,7 @@ public String getJobLocalDir() {
public long getMemoryForMapTask() { public long getMemoryForMapTask() {
long value = getDeprecatedMemoryValue(); long value = getDeprecatedMemoryValue();
if (value < 0) { if (value < 0) {
return getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, return getMemoryRequired(TaskType.MAP);
JobContext.DEFAULT_MAP_MEMORY_MB);
} }
return value; return value;
} }
@ -1853,8 +1852,7 @@ public void setMemoryForMapTask(long mem) {
public long getMemoryForReduceTask() { public long getMemoryForReduceTask() {
long value = getDeprecatedMemoryValue(); long value = getDeprecatedMemoryValue();
if (value < 0) { if (value < 0) {
return getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, return getMemoryRequired(TaskType.REDUCE);
JobContext.DEFAULT_REDUCE_MEMORY_MB);
} }
return value; return value;
} }

19
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
View File

@ -19,9 +19,7 @@
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.MRJobConfig;
public class TestJobConf { public class TestJobConf {
@ -87,8 +85,10 @@ public void testMemoryConfigForMapOrReduceTask(){
configuration.set("mapred.task.maxvmem" , String.valueOf(-1)); configuration.set("mapred.task.maxvmem" , String.valueOf(-1));
configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1"); configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1"); configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
Assert.assertEquals(configuration.getMemoryForMapTask(),-1); Assert.assertEquals(configuration.getMemoryForMapTask(),
Assert.assertEquals(configuration.getMemoryForReduceTask(),-1); MRJobConfig.DEFAULT_MAP_MEMORY_MB);
Assert.assertEquals(configuration.getMemoryForReduceTask(),
MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
configuration = new JobConf(); configuration = new JobConf();
configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024)); configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
@ -96,7 +96,6 @@ public void testMemoryConfigForMapOrReduceTask(){
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "3"); configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "3");
Assert.assertEquals(configuration.getMemoryForMapTask(),2); Assert.assertEquals(configuration.getMemoryForMapTask(),2);
Assert.assertEquals(configuration.getMemoryForReduceTask(),2); Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
} }
/** /**
@ -129,8 +128,10 @@ public void testNegativeValuesForMemoryParams() {
configuration.set(MRJobConfig.MAP_MEMORY_MB, "-5"); configuration.set(MRJobConfig.MAP_MEMORY_MB, "-5");
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "-6"); configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "-6");
Assert.assertEquals(-5, configuration.getMemoryForMapTask()); Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
Assert.assertEquals(-6, configuration.getMemoryForReduceTask()); configuration.getMemoryForMapTask());
Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
configuration.getMemoryForReduceTask());
} }
/** /**
@ -144,13 +145,13 @@ public void testMaxVirtualMemoryForTask() {
configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300)); configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1)); configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
Assert.assertEquals( Assert.assertEquals(
configuration.getMaxVirtualMemoryForTask(), 300 * 1024 * 1024); configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
configuration = new JobConf(); configuration = new JobConf();
configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1)); configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(200)); configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(200));
Assert.assertEquals( Assert.assertEquals(
configuration.getMaxVirtualMemoryForTask(), 200 * 1024 * 1024); configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
configuration = new JobConf(); configuration = new JobConf();
configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1)); configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));