big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

MAPREDUCE-5145. Changed default max-attempts to be more than one for MR jobs inline with YARN. Contributed by Zhijie Shen.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1477889 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-05-01 00:42:32 +00:00
parent 0728f00282
commit 71be8ad325
4 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -215,6 +215,9 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-5069. add concrete common implementations of MAPREDUCE-5069. add concrete common implementations of
CombineFileInputFormat (Sangjin Lee via bobby) CombineFileInputFormat (Sangjin Lee via bobby)
MAPREDUCE-5145. Changed default max-attempts to be more than one for MR jobs
inline with YARN. (Zhijie Shen via vinodkv)
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method MAPREDUCE-4974. Optimising the LineRecordReader initialize() method

9
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
View File

@ -90,6 +90,7 @@ import org.junit.Test;
JobId jobid = recordFactory.newRecordInstance(JobId.class); JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId); jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class); ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
appMaster.init(conf); appMaster.init(conf);
@ -116,8 +117,9 @@ import org.junit.Test;
appId.setId(0); appId.setId(0);
attemptId.setApplicationId(appId); attemptId.setApplicationId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class); ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
JobStateInternal.REBOOT, 4); JobStateInternal.REBOOT, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
appMaster.init(conf); appMaster.init(conf);
appMaster.start(); appMaster.start();
//shutdown the job, not the lastRetry //shutdown the job, not the lastRetry
@ -144,7 +146,7 @@ import org.junit.Test;
attemptId.setApplicationId(appId); attemptId.setApplicationId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class); ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
JobStateInternal.REBOOT, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); JobStateInternal.REBOOT, 1); //no retry
appMaster.init(conf); appMaster.init(conf);
appMaster.start(); appMaster.start();
//shutdown the job, is lastRetry //shutdown the job, is lastRetry
@ -201,8 +203,7 @@ import org.junit.Test;
JobId jobid = recordFactory.newRecordInstance(JobId.class); JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId); jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class); ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, 1); //no retry
MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
appMaster.init(conf); appMaster.init(conf);
//simulate the process being killed //simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook = MRAppMaster.MRAppMasterShutdownHook hook =

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -677,6 +677,6 @@ public interface MRJobConfig {
*/ */
public static final String MR_AM_MAX_ATTEMPTS = "mapreduce.am.max-attempts"; public static final String MR_AM_MAX_ATTEMPTS = "mapreduce.am.max-attempts";
public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 1; public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 2;
} }

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
View File

@ -816,10 +816,11 @@
<property> <property>
<name>mapreduce.am.max-attempts</name> <name>mapreduce.am.max-attempts</name>
<value>1</value> <value>2</value>
<description>The maximum number of application attempts. It is a <description>The maximum number of application attempts. It is a
application-specific setting. It should not be larger than the global number application-specific setting. It should not be larger than the global number
set by resourcemanager. Otherwise, it will be override.</description> set by resourcemanager. Otherwise, it will be override. The default number is
set to 2, to allow at least one retry for AM.</description>
</property> </property>
<!-- Job Notification Configuration --> <!-- Job Notification Configuration -->