big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6223. TestJobConf#testNegativeValueForTaskVmem failures. (Varun Saxena via kasha)

Browse Source
This commit is contained in:
Karthik Kambatla 2015-02-26 14:24:19 -08:00
parent c6d5b37a1c
commit 1047c883ba
3 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -390,6 +390,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-2815. JavaDoc does not generate correctly for
MultithreadedMapRunner. (Chris Palmer via aajisaka)
MAPREDUCE-6223. TestJobConf#testNegativeValueForTaskVmem failures.
(Varun Saxena via kasha)
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
View File

@ -1826,8 +1826,7 @@ public String getJobLocalDir() {
public long getMemoryForMapTask() {
long value = getDeprecatedMemoryValue();
if (value < 0) {
return getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
JobContext.DEFAULT_MAP_MEMORY_MB);
return getMemoryRequired(TaskType.MAP);
}
return value;
}
@ -1853,8 +1852,7 @@ public void setMemoryForMapTask(long mem) {
public long getMemoryForReduceTask() {
long value = getDeprecatedMemoryValue();
if (value < 0) {
return getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
JobContext.DEFAULT_REDUCE_MEMORY_MB);
return getMemoryRequired(TaskType.REDUCE);
}
return value;
}

23
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
View File

@ -19,9 +19,7 @@
import org.junit.Assert;
import org.junit.Test;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapreduce.MRJobConfig;
public class TestJobConf {
@ -87,8 +85,10 @@ public void testMemoryConfigForMapOrReduceTask(){
configuration.set("mapred.task.maxvmem" , String.valueOf(-1));
configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
Assert.assertEquals(configuration.getMemoryForMapTask(),-1);
Assert.assertEquals(configuration.getMemoryForReduceTask(),-1);
Assert.assertEquals(configuration.getMemoryForMapTask(),
MRJobConfig.DEFAULT_MAP_MEMORY_MB);
Assert.assertEquals(configuration.getMemoryForReduceTask(),
MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
configuration = new JobConf();
configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
@ -96,7 +96,6 @@ public void testMemoryConfigForMapOrReduceTask(){
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "3");
Assert.assertEquals(configuration.getMemoryForMapTask(),2);
Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
}
/**
@ -106,7 +105,7 @@ public void testMemoryConfigForMapOrReduceTask(){
@Test
public void testNegativeValueForTaskVmem() {
JobConf configuration = new JobConf();
configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3");
Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
configuration.getMemoryForMapTask());
@ -129,10 +128,12 @@ public void testNegativeValuesForMemoryParams() {
configuration.set(MRJobConfig.MAP_MEMORY_MB, "-5");
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "-6");
Assert.assertEquals(-5, configuration.getMemoryForMapTask());
Assert.assertEquals(-6, configuration.getMemoryForReduceTask());
Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
configuration.getMemoryForMapTask());
Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
configuration.getMemoryForReduceTask());
}
/**
* Test deprecated accessor and mutator method for mapred.task.maxvmem
*/
@ -144,13 +145,13 @@ public void testMaxVirtualMemoryForTask() {
configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
Assert.assertEquals(
configuration.getMaxVirtualMemoryForTask(), 300 * 1024 * 1024);
configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
configuration = new JobConf();
configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(200));
Assert.assertEquals(
configuration.getMaxVirtualMemoryForTask(), 200 * 1024 * 1024);
configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
configuration = new JobConf();
configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));