diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 2668ce9222..98268d39ad 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -490,6 +490,9 @@ Release 2.1.0-beta - UNRELEASED MAPREDUCE-5296. Fix backward incompatibility for JobControl by adding the omitted addJob. (Zhijie Shen via acmurthy) + MAPREDUCE-5245. Added back constants to JobConf to fix incompatibilities. + (Zhijie Shen via acmurthy) + BREAKDOWN OF HADOOP-8562 SUBTASKS MAPREDUCE-4739. Some MapReduce tests fail to find winutils. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index a6ae5e2da4..9eee571975 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -20,9 +20,6 @@ import java.io.IOException; -import java.net.URL; -import java.net.URLDecoder; -import java.util.Enumeration; import java.util.regex.Pattern; import org.apache.commons.logging.Log; @@ -122,8 +119,8 @@ public class JobConf extends Configuration { } /** - * @deprecated Use {@link #MAPRED_JOB_MAP_MEMORY_MB_PROPERTY} and - * {@link #MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY} + * @deprecated Use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} and + * {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY} */ @Deprecated public static final String MAPRED_TASK_MAXVMEM_PROPERTY = @@ -167,12 +164,28 @@ public class JobConf extends Configuration { */ public static final String DEFAULT_QUEUE_NAME = "default"; - static final String MAPRED_JOB_MAP_MEMORY_MB_PROPERTY = + static final String MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY = JobContext.MAP_MEMORY_MB; - static final String MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY = + static final String MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY = JobContext.REDUCE_MEMORY_MB; + /** + * The variable is kept for M/R 1.x applications, while M/R 2.x applications + * should use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} + */ + @Deprecated + public static final String MAPRED_JOB_MAP_MEMORY_MB_PROPERTY = + "mapred.job.map.memory.mb"; + + /** + * The variable is kept for M/R 1.x applications, while M/R 2.x applications + * should use {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY} + */ + @Deprecated + public static final String MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY = + "mapred.job.reduce.memory.mb"; + /** Pattern for the default unpacking behavior for job jars */ public static final Pattern UNPACK_JAR_PATTERN_DEFAULT = Pattern.compile("(?:classes/|lib/).*"); @@ -335,8 +348,67 @@ public class JobConf extends Configuration { * Default logging level for map/reduce tasks. */ public static final Level DEFAULT_LOG_LEVEL = Level.INFO; - - + + /** + * The variable is kept for M/R 1.x applications, M/R 2.x applications should + * use {@link MRJobConfig#WORKFLOW_ID} instead + */ + @Deprecated + public static final String WORKFLOW_ID = MRJobConfig.WORKFLOW_ID; + + /** + * The variable is kept for M/R 1.x applications, M/R 2.x applications should + * use {@link MRJobConfig#WORKFLOW_NAME} instead + */ + @Deprecated + public static final String WORKFLOW_NAME = MRJobConfig.WORKFLOW_NAME; + + /** + * The variable is kept for M/R 1.x applications, M/R 2.x applications should + * use {@link MRJobConfig#WORKFLOW_NODE_NAME} instead + */ + @Deprecated + public static final String WORKFLOW_NODE_NAME = + MRJobConfig.WORKFLOW_NODE_NAME; + + /** + * The variable is kept for M/R 1.x applications, M/R 2.x applications should + * use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_STRING} instead + */ + @Deprecated + public static final String WORKFLOW_ADJACENCY_PREFIX_STRING = + MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_STRING; + + /** + * The variable is kept for M/R 1.x applications, M/R 2.x applications should + * use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_PATTERN} instead + */ + @Deprecated + public static final String WORKFLOW_ADJACENCY_PREFIX_PATTERN = + MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_PATTERN; + + /** + * The variable is kept for M/R 1.x applications, M/R 2.x applications should + * use {@link MRJobConfig#WORKFLOW_TAGS} instead + */ + @Deprecated + public static final String WORKFLOW_TAGS = MRJobConfig.WORKFLOW_TAGS; + + /** + * The variable is kept for M/R 1.x applications, M/R 2.x applications should + * not use it + */ + @Deprecated + public static final String MAPREDUCE_RECOVER_JOB = + "mapreduce.job.restart.recover"; + + /** + * The variable is kept for M/R 1.x applications, M/R 2.x applications should + * not use it + */ + @Deprecated + public static final boolean DEFAULT_MAPREDUCE_RECOVER_JOB = true; + /** * Construct a map/reduce job configuration. */ @@ -1695,6 +1767,12 @@ public String getJobLocalDir() { */ public long getMemoryForMapTask() { long value = getDeprecatedMemoryValue(); + if (value == DISABLED_MEMORY_LIMIT) { + value = normalizeMemoryConfigValue( + getLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, + DISABLED_MEMORY_LIMIT)); + } + // In case that M/R 1.x applications use the old property name if (value == DISABLED_MEMORY_LIMIT) { value = normalizeMemoryConfigValue( getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, @@ -1704,6 +1782,8 @@ public long getMemoryForMapTask() { } public void setMemoryForMapTask(long mem) { + setLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, mem); + // In case that M/R 1.x applications use the old property name setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, mem); } @@ -1722,6 +1802,12 @@ public void setMemoryForMapTask(long mem) { */ public long getMemoryForReduceTask() { long value = getDeprecatedMemoryValue(); + if (value == DISABLED_MEMORY_LIMIT) { + value = normalizeMemoryConfigValue( + getLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, + DISABLED_MEMORY_LIMIT)); + } + // In case that M/R 1.x applications use the old property name if (value == DISABLED_MEMORY_LIMIT) { value = normalizeMemoryConfigValue( getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, @@ -1745,6 +1831,8 @@ private long getDeprecatedMemoryValue() { } public void setMemoryForReduceTask(long mem) { + setLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, mem); + // In case that M/R 1.x applications use the old property name setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, mem); } @@ -1927,8 +2015,8 @@ static String deprecatedString(String key) { private void checkAndWarnDeprecation() { if(get(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) != null) { LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) - + " Instead use " + JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY - + " and " + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY); + + " Instead use " + JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY + + " and " + JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY); } if(get(JobConf.MAPRED_TASK_ULIMIT) != null ) { LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_ULIMIT)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java index 646a747aa5..f4327459e6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java @@ -22,6 +22,7 @@ import static org.junit.Assert.*; import org.apache.hadoop.fs.Path; +import org.junit.Assert; import org.junit.Test; /** @@ -152,4 +153,36 @@ public void testJobConf() { JobConf.deprecatedString("key")); } + + /** + * Ensure that M/R 1.x applications can get and set task virtual memory with + * old property names + */ + @SuppressWarnings("deprecation") + @Test (timeout = 1000) + public void testDeprecatedPropertyNameForTaskVmem() { + JobConf configuration = new JobConf(); + + configuration.setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, 1024); + configuration.setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, 1024); + Assert.assertEquals(1024, configuration.getMemoryForMapTask()); + Assert.assertEquals(1024, configuration.getMemoryForReduceTask()); + // Make sure new property names aren't broken by the old ones + configuration.setLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, 1025); + configuration.setLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, 1025); + Assert.assertEquals(1025, configuration.getMemoryForMapTask()); + Assert.assertEquals(1025, configuration.getMemoryForReduceTask()); + + configuration.setMemoryForMapTask(2048); + configuration.setMemoryForReduceTask(2048); + Assert.assertEquals(2048, configuration.getLong( + JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, -1)); + Assert.assertEquals(2048, configuration.getLong( + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, -1)); + // Make sure new property names aren't broken by the old ones + Assert.assertEquals(2048, configuration.getLong( + JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, -1)); + Assert.assertEquals(2048, configuration.getLong( + JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, -1)); + } }