MAPREDUCE-6239. Consolidate TestJobConf classes in hadoop-mapreduce-client-jobclient and hadoop-mapreduce-client-core. Contributed by Varun Saxena.

hadoop-mapreduce-project/CHANGES.txt

@@ -256,6 +256,10 @@ Release 2.8.0 - UNRELEASED
 
   IMPROVEMENTS
 
+    MAPREDUCE-6239. Consolidate TestJobConf classes in
+    hadoop-mapreduce-client-jobclient and hadoop-mapreduce-client-core
+    (Varun Saxena via harsh)
+
     MAPREDUCE-5807. Print usage by TeraSort job. (Rohith via harsh)
 
     MAPREDUCE-4653. TestRandomAlgorithm has an unused "import" statement.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java

@@ -22,6 +22,7 @@
 import static org.junit.Assert.*;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -188,4 +189,176 @@ public void testDeprecatedPropertyNameForTaskVmem() {
     Assert.assertEquals(2048, configuration.getLong(
         JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, -1));
   }
+
+
+  @Test
+  public void testProfileParamsDefaults() {
+    JobConf configuration = new JobConf();
+    String result = configuration.getProfileParams();
+    Assert.assertNotNull(result);
+    Assert.assertTrue(result.contains("file=%s"));
+    Assert.assertTrue(result.startsWith("-agentlib:hprof"));
+  }
+
+  @Test
+  public void testProfileParamsSetter() {
+    JobConf configuration = new JobConf();
+
+    configuration.setProfileParams("test");
+    Assert.assertEquals("test", configuration.get(MRJobConfig.TASK_PROFILE_PARAMS));
+  }
+
+  @Test
+  public void testProfileParamsGetter() {
+    JobConf configuration = new JobConf();
+
+    configuration.set(MRJobConfig.TASK_PROFILE_PARAMS, "test");
+    Assert.assertEquals("test", configuration.getProfileParams());
+  }
+
+  /**
+   * Testing mapred.task.maxvmem replacement with new values
+   *
+   */
+  @Test
+  public void testMemoryConfigForMapOrReduceTask(){
+    JobConf configuration = new JobConf();
+    configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(300));
+    Assert.assertEquals(configuration.getMemoryForMapTask(),300);
+    Assert.assertEquals(configuration.getMemoryForReduceTask(),300);
+
+    configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+    configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(300));
+    Assert.assertEquals(configuration.getMemoryForMapTask(),2);
+    Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+
+    configuration = new JobConf();
+    configuration.set("mapred.task.maxvmem" , "-1");
+    configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(400));
+    Assert.assertEquals(configuration.getMemoryForMapTask(), 300);
+    Assert.assertEquals(configuration.getMemoryForReduceTask(), 400);
+
+    configuration = new JobConf();
+    configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+    configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
+    Assert.assertEquals(configuration.getMemoryForMapTask(),2);
+    Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+
+    configuration = new JobConf();
+    configuration.set("mapred.task.maxvmem" , String.valueOf(-1));
+    configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
+    Assert.assertEquals(configuration.getMemoryForMapTask(),
+        MRJobConfig.DEFAULT_MAP_MEMORY_MB);
+    Assert.assertEquals(configuration.getMemoryForReduceTask(),
+        MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
+
+    configuration = new JobConf();
+    configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+    configuration.set(MRJobConfig.MAP_MEMORY_MB, "3");
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "3");
+    Assert.assertEquals(configuration.getMemoryForMapTask(),2);
+    Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+  }
+
+  /**
+   * Test that negative values for MAPRED_TASK_MAXVMEM_PROPERTY cause
+   * new configuration keys' values to be used.
+   */
+  @Test
+  public void testNegativeValueForTaskVmem() {
+    JobConf configuration = new JobConf();
+
+    configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3");
+    Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
+        configuration.getMemoryForMapTask());
+    Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
+        configuration.getMemoryForReduceTask());
+
+    configuration.set(MRJobConfig.MAP_MEMORY_MB, "4");
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "5");
+    Assert.assertEquals(4, configuration.getMemoryForMapTask());
+    Assert.assertEquals(5, configuration.getMemoryForReduceTask());
+
+  }
+
+  /**
+   * Test that negative values for new configuration keys get passed through.
+   */
+  @Test
+  public void testNegativeValuesForMemoryParams() {
+    JobConf configuration = new JobConf();
+
+    configuration.set(MRJobConfig.MAP_MEMORY_MB, "-5");
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "-6");
+    Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
+        configuration.getMemoryForMapTask());
+    Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
+        configuration.getMemoryForReduceTask());
+  }
+
+  /**
+   *   Test deprecated accessor and mutator method for mapred.task.maxvmem
+   */
+  @Test
+  public void testMaxVirtualMemoryForTask() {
+    JobConf configuration = new JobConf();
+
+    //get test case
+    configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
+    Assert.assertEquals(
+      configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
+
+    configuration = new JobConf();
+    configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(200));
+    Assert.assertEquals(
+      configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
+
+    configuration = new JobConf();
+    configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
+    configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
+    Assert.assertEquals(
+      configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
+
+    configuration = new JobConf();
+    configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
+    Assert.assertEquals(
+      configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
+
+    //set test case
+
+    configuration = new JobConf();
+    configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
+    Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
+    Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
+
+    configuration = new JobConf();
+    configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
+    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(400));
+    configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
+    Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
+    Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
+  }
+
+  /**
+   * Ensure that by default JobContext.MAX_TASK_FAILURES_PER_TRACKER is less
+   * JobContext.MAP_MAX_ATTEMPTS and JobContext.REDUCE_MAX_ATTEMPTS so that
+   * failed tasks will be retried on other nodes
+   */
+  @Test
+  public void testMaxTaskFailuresPerTracker() {
+    JobConf jobConf = new JobConf(true);
+    Assert.assertTrue("By default JobContext.MAX_TASK_FAILURES_PER_TRACKER was "
+      + "not less than JobContext.MAP_MAX_ATTEMPTS and REDUCE_MAX_ATTEMPTS"
+      ,jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxMapAttempts() &&
+      jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxReduceAttempts()
+      );
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java

@@ -1,199 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.conf;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-
-public class TestJobConf {
-
-  @Test
-  public void testProfileParamsDefaults() {
-    JobConf configuration = new JobConf();
-    String result = configuration.getProfileParams();
-    Assert.assertNotNull(result);
-    Assert.assertTrue(result.contains("file=%s"));
-    Assert.assertTrue(result.startsWith("-agentlib:hprof"));
-  }
-
-  @Test
-  public void testProfileParamsSetter() {
-    JobConf configuration = new JobConf();
-
-    configuration.setProfileParams("test");
-    Assert.assertEquals("test", configuration.get(MRJobConfig.TASK_PROFILE_PARAMS));
-  }
-
-  @Test
-  public void testProfileParamsGetter() {
-    JobConf configuration = new JobConf();
-
-    configuration.set(MRJobConfig.TASK_PROFILE_PARAMS, "test");
-    Assert.assertEquals("test", configuration.getProfileParams());
-  }
-
-  /**
-   * Testing mapred.task.maxvmem replacement with new values
-   *
-   */
-  @Test
-  public void testMemoryConfigForMapOrReduceTask(){
-    JobConf configuration = new JobConf();
-    configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(300));
-    Assert.assertEquals(configuration.getMemoryForMapTask(),300);
-    Assert.assertEquals(configuration.getMemoryForReduceTask(),300);
-
-    configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
-    configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(300));
-    Assert.assertEquals(configuration.getMemoryForMapTask(),2);
-    Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
-
-    configuration = new JobConf();
-    configuration.set("mapred.task.maxvmem" , "-1");
-    configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(400));
-    Assert.assertEquals(configuration.getMemoryForMapTask(), 300);
-    Assert.assertEquals(configuration.getMemoryForReduceTask(), 400);
-
-    configuration = new JobConf();
-    configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
-    configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
-    Assert.assertEquals(configuration.getMemoryForMapTask(),2);
-    Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
-
-    configuration = new JobConf();
-    configuration.set("mapred.task.maxvmem" , String.valueOf(-1));
-    configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
-    Assert.assertEquals(configuration.getMemoryForMapTask(),
-        MRJobConfig.DEFAULT_MAP_MEMORY_MB);
-    Assert.assertEquals(configuration.getMemoryForReduceTask(),
-        MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
-
-    configuration = new JobConf();
-    configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
-    configuration.set(MRJobConfig.MAP_MEMORY_MB, "3");
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "3");
-    Assert.assertEquals(configuration.getMemoryForMapTask(),2);
-    Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
-  }
-
-  /**
-   * Test that negative values for MAPRED_TASK_MAXVMEM_PROPERTY cause
-   * new configuration keys' values to be used.
-   */
-  @Test
-  public void testNegativeValueForTaskVmem() {
-    JobConf configuration = new JobConf();
-
-    configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3");
-    Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
-        configuration.getMemoryForMapTask());
-    Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
-        configuration.getMemoryForReduceTask());
-    
-    configuration.set(MRJobConfig.MAP_MEMORY_MB, "4");
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "5");
-    Assert.assertEquals(4, configuration.getMemoryForMapTask());
-    Assert.assertEquals(5, configuration.getMemoryForReduceTask());
-    
-  }
-  
-  /**
-   * Test that negative values for new configuration keys get passed through.
-   */
-  @Test
-  public void testNegativeValuesForMemoryParams() {
-    JobConf configuration = new JobConf();
-        
-    configuration.set(MRJobConfig.MAP_MEMORY_MB, "-5");
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "-6");
-    Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
-        configuration.getMemoryForMapTask());
-    Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
-        configuration.getMemoryForReduceTask());
-  }
-
-  /**
-   *   Test deprecated accessor and mutator method for mapred.task.maxvmem
-   */
-  @Test
-  public void testMaxVirtualMemoryForTask() {
-    JobConf configuration = new JobConf();
-
-    //get test case
-    configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
-    Assert.assertEquals(
-      configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
-
-    configuration = new JobConf();
-    configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(200));
-    Assert.assertEquals(
-      configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
-
-    configuration = new JobConf();
-    configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
-    configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
-    Assert.assertEquals(
-      configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
-
-    configuration = new JobConf();
-    configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
-    Assert.assertEquals(
-      configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
-
-    //set test case
-
-    configuration = new JobConf();
-    configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
-    Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
-    Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
-
-    configuration = new JobConf();   
-    configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
-    configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(400));
-    configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
-    Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
-    Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
-    
-    
-  }
-
-  /**
-   * Ensure that by default JobContext.MAX_TASK_FAILURES_PER_TRACKER is less
-   * JobContext.MAP_MAX_ATTEMPTS and JobContext.REDUCE_MAX_ATTEMPTS so that
-   * failed tasks will be retried on other nodes
-   */
-  @Test
-  public void testMaxTaskFailuresPerTracker() {
-    JobConf jobConf = new JobConf(true);
-    Assert.assertTrue("By default JobContext.MAX_TASK_FAILURES_PER_TRACKER was "
-      + "not less than JobContext.MAP_MAX_ATTEMPTS and REDUCE_MAX_ATTEMPTS"
-      ,jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxMapAttempts() &&
-      jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxReduceAttempts()
-      );
-  }
-}