big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5420. Remove mapreduce.task.tmp.dir from mapred-default.xml. Contributed by James Carman. (harsh)

Browse Source
This commit is contained in:
Harsh J 2014-12-08 17:34:39 +05:30
parent 9a44db48b4
commit cb99f43305
6 changed files with 4 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -235,6 +235,9 @@ Release 2.7.0 - UNRELEASED
IMPROVEMENTS
MAPREDUCE-5420. Remove mapreduce.task.tmp.dir from mapred-default.xml
(James Carman via harsh)
MAPREDUCE-5932. Provide an option to use a dedicated reduce-side shuffle
log (Gera Shegalov via jlowe)

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -166,8 +166,6 @@ public interface MRJobConfig {
public static final String PRESERVE_FILES_PATTERN = "mapreduce.task.files.preserve.filepattern";
public static final String TASK_TEMP_DIR = "mapreduce.task.tmp.dir";
public static final String TASK_DEBUGOUT_LINES = "mapreduce.task.debugout.lines";
public static final String RECORDS_BEFORE_PROGRESS = "mapreduce.task.merge.progress.records";

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
View File

@ -274,8 +274,6 @@ private static void addDeprecatedKeys() {
MRJobConfig.PRESERVE_FAILED_TASK_FILES),
new DeprecationDelta("keep.task.files.pattern",
MRJobConfig.PRESERVE_FILES_PATTERN),
new DeprecationDelta("mapred.child.tmp",
MRJobConfig.TASK_TEMP_DIR),
new DeprecationDelta("mapred.debug.out.lines",
MRJobConfig.TASK_DEBUGOUT_LINES),
new DeprecationDelta("mapred.merge.recordsBeforeProgress",

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
View File

@ -319,18 +319,6 @@
</description>
</property>
<property>
<name>mapreduce.task.tmp.dir</name>
<value>./tmp</value>
<description> To set the value of tmp directory for map and reduce tasks.
If the value is an absolute path, it is directly assigned. Otherwise, it is
prepended with task's working directory. The java tasks are executed with
option -Djava.io.tmpdir='the absolute path of the tmp dir'. Pipes and
streaming are set with environment variable,
TMPDIR='the absolute path of the tmp dir'
</description>
</property>
<property>
<name>mapreduce.map.log.level</name>
<value>INFO</value>

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
View File

@ -166,7 +166,6 @@
<property><!--Loaded from mapred-default.xml--><name>mapreduce.job.end-notification.max.attempts</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobhistory.max-age-ms</name><value>10000000000</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.zookeeper-store.session.timeout-ms</name><value>60000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.tmp.dir</name><value>./tmp</value></property>
<property><!--Loaded from job.xml--><name>dfs.default.chunk.view.size</name><value>32768</value></property>
<property><!--Loaded from job.xml--><name>kfs.bytes-per-checksum</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.memory.mb</name><value>512</value></property>

31
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
View File

@ -391,36 +391,7 @@ public static void tearDown() {
ioe.printStackTrace();
}
}
/**
* Tests task's temp directory.
*
* In this test, we give different values to mapreduce.task.tmp.dir
* both relative and absolute. And check whether the temp directory
* is created. We also check whether java.io.tmpdir value is same as
* the directory specified. We create a temp file and check if is is
* created in the directory specified.
*/
@Test
public void testTaskTempDir(){
try {
JobConf conf = new JobConf(mr.getConfig());
// intialize input, output directories
Path inDir = new Path("testing/wc/input");
Path outDir = new Path("testing/wc/output");
String input = "The input";
configure(conf, inDir, outDir, input,
MapClass.class, IdentityReducer.class);
launchTest(conf, inDir, outDir, input);
} catch(Exception e) {
e.printStackTrace();
fail("Exception in testing temp dir");
tearDown();
}
}
/**
* To test OS dependent setting of default execution path for a MapRed task.
* Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set -