diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8f42562ce8..de9e597170 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -175,6 +175,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4520. Added support for MapReduce applications to request for CPU cores along-with memory post YARN-2. (acmurthy) + MAPREDUCE-4810. Added new admin command options for MR AM. (Jerry Chen via + vinodkv) + IMPROVEMENTS MAPREDUCE-3678. The Map tasks logs should have the value of input diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index defd06dc2a..f4ff04b111 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -365,6 +365,11 @@ public interface MRJobConfig { MR_AM_PREFIX+"command-opts"; public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1024m"; + /** Admin command opts passed to the MR app master.*/ + public static final String MR_AM_ADMIN_COMMAND_OPTS = + MR_AM_PREFIX+"admin-command-opts"; + public static final String DEFAULT_MR_AM_ADMIN_COMMAND_OPTS = ""; + /** Root Logging level passed to the MR app master.*/ public static final String MR_AM_LOG_LEVEL = MR_AM_PREFIX+"log.level"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 8b86f5706a..a08e261b00 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -874,6 +874,20 @@ + + yarn.app.mapreduce.am.admin-command-opts + + Java opts for the MR App Master processes for admin purposes. + It will appears before the opts set by yarn.app.mapreduce.am.command-opts and + thus its options can be overridden user. + + Usage of -Djava.library.path can cause programs to no longer function if + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. + + + yarn.app.mapreduce.am.job.task.listener.thread-count 30 diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index c3f6b75b75..98c94fab80 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -394,6 +394,11 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL); MRApps.addLog4jSystemProperties(logLevel, logSize, vargs); + // Add AM admin command opts before user command opts + // so that it can be overridden by user + vargs.add(conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, + MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS)); + vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java index 3bbd1324bd..aa844c025f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java @@ -29,6 +29,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.List; import junit.framework.TestCase; @@ -39,11 +40,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.ClientCache; import org.apache.hadoop.mapred.ClientServiceDelegate; +import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.ResourceMgrDelegate; import org.apache.hadoop.mapred.YARNRunner; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobPriority; import org.apache.hadoop.mapreduce.JobStatus.State; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.ClientRMProtocol; @@ -65,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -240,4 +244,68 @@ public synchronized void start() { delegate.getQueueAclsForCurrentUser(); verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class)); } + + @Test + public void testAMAdminCommandOpts() throws Exception { + JobConf jobConf = new JobConf(); + + jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true"); + jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m"); + + YARNRunner yarnRunner = new YARNRunner(jobConf); + + File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE); + OutputStream out = new FileOutputStream(jobxml); + conf.writeXml(out); + out.close(); + + File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT); + out = new FileOutputStream(jobsplit); + out.close(); + + File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO); + out = new FileOutputStream(jobsplitmetainfo); + out.close(); + + File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE); + out = new FileOutputStream(appTokens); + out.close(); + + ApplicationSubmissionContext submissionContext = + yarnRunner.createApplicationSubmissionContext(jobConf, testWorkDir.toString(), new Credentials()); + + ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec(); + List commands = containerSpec.getCommands(); + + int index = 0; + int adminIndex = 0; + int adminPos = -1; + int userIndex = 0; + int userPos = -1; + + for(String command : commands) { + if(command != null) { + adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true"); + if(adminPos >= 0) + adminIndex = index; + + userPos = command.indexOf("-Xmx1024m"); + if(userPos >= 0) + userIndex = index; + } + + index++; + } + + // Check both admin java opts and user java opts are in the commands + assertTrue("AM admin command opts not in the commands.", adminPos > 0); + assertTrue("AM user command opts not in the commands.", userPos > 0); + + // Check the admin java opts is before user java opts in the commands + if(adminIndex == userIndex) { + assertTrue("AM admin command opts is after user command opts.", adminPos < userPos); + } else { + assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex); + } + } }