From 88c7fdd06599671bdd519b0911c984410ecbe3fc Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Fri, 18 Jan 2013 22:41:19 +0000 Subject: [PATCH] MAPREDUCE-4458. Warn if java.library.path is used for AM or Task (Robert Parker via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1435386 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../apache/hadoop/mapreduce/MRJobConfig.java | 3 + .../src/main/resources/mapred-default.xml | 11 ++++ .../org/apache/hadoop/mapred/YARNRunner.java | 47 +++++++++++--- .../apache/hadoop/mapred/TestYARNRunner.java | 62 ++++++++++++++++++- 5 files changed, 116 insertions(+), 10 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 19ef075f91..a83d9787d6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -652,6 +652,9 @@ Release 0.23.7 - UNRELEASED BUG FIXES + MAPREDUCE-4458. Warn if java.library.path is used for AM or Task + (Robert Parker via jeagles) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 5b15467160..5fc7144a8c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -530,6 +530,9 @@ public interface MRJobConfig { public static final String MR_AM_ENV = MR_AM_PREFIX + "env"; + public static final String MR_AM_ADMIN_USER_ENV = + MR_AM_PREFIX + "admin.user.env"; + public static final String MAPRED_MAP_ADMIN_JAVA_OPTS = "mapreduce.admin.map.child.java.opts"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index bf575bc981..e756860cad 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -857,6 +857,17 @@ + + yarn.app.mapreduce.am.admin.user.env + + Environment variables for the MR App Master + processes for admin purposes. These values are set first and can be + overridden by the user env (yarn.app.mapreduce.am.env) Example : + 1) A=foo This will set the env variable A to foo + 2) B=$B:c This is inherit app master's B env variable. + + + yarn.app.mapreduce.am.command-opts -Xmx1024m diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 427f5a03f8..c401b93c96 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -101,7 +101,7 @@ public class YARNRunner implements ClientProtocol { private Configuration conf; private final FileContext defaultFileContext; - /* usually is false unless the jobclient getdelegation token is + /* usually is false unless the jobclient get delegation token is * called. This is a hack wherein we do return a token from RM * on getDelegationtoken but due to the restricted api on jobclient * we just add a job history DT token when submitting a job. @@ -165,12 +165,12 @@ public void cancelDelegationToken(Token arg0) @Override public TaskTrackerInfo[] getActiveTrackers() throws IOException, InterruptedException { - return resMgrDelegate.getActiveTrackers(); + return resMgrDelegate.getActiveTrackers(); } @Override public JobStatus[] getAllJobs() throws IOException, InterruptedException { - return resMgrDelegate.getAllJobs(); + return resMgrDelegate.getAllJobs(); } @Override @@ -394,14 +394,31 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL); MRApps.addLog4jSystemProperties(logLevel, logSize, vargs); + // Check for Java Lib Path usage in MAP and REDUCE configs + warnForJavaLibPath(conf.get(MRJobConfig.MAP_JAVA_OPTS,""), "map", + MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.MAP_ENV); + warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,""), "map", + MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV); + warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce", + MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV); + warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce", + MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV); + // Add AM admin command opts before user command opts // so that it can be overridden by user - vargs.add(conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, - MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS)); + String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, + MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS); + warnForJavaLibPath(mrAppMasterAdminOptions, "app master", + MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, MRJobConfig.MR_AM_ADMIN_USER_ENV); + vargs.add(mrAppMasterAdminOptions); + + // Add AM user command opts + String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS, + MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS); + warnForJavaLibPath(mrAppMasterUserOptions, "app master", + MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV); + vargs.add(mrAppMasterUserOptions); - vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS, - MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS)); - vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + ApplicationConstants.STDOUT); @@ -425,6 +442,9 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( Map environment = new HashMap(); MRApps.setClasspath(environment, conf); + // Setup the environment variables for Admin first + MRApps.setEnvFromInputString(environment, + conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV)); // Setup the environment variables (LD_LIBRARY_PATH, etc) MRApps.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ENV)); @@ -582,4 +602,15 @@ public LogParams getLogFileParams(JobID jobID, TaskAttemptID taskAttemptID) throws IOException { return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID); } + + private static void warnForJavaLibPath(String opts, String component, + String javaConf, String envConf) { + if (opts != null && opts.contains("-Djava.library.path")) { + LOG.warn("Usage of -Djava.library.path in " + javaConf + " can cause " + + "programs to no longer function if hadoop native libraries " + + "are used. These values should be set as part of the " + + "LD_LIBRARY_PATH in the " + component + " JVM env using " + + envConf + " config settings."); + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index f3c24c65c0..5675742cfd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -83,6 +84,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.log4j.Appender; +import org.apache.log4j.Layout; +import org.apache.log4j.Logger; +import org.apache.log4j.SimpleLayout; +import org.apache.log4j.WriterAppender; import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; @@ -189,7 +195,7 @@ public void testJobSubmissionFailure() throws Exception { @Test public void testResourceMgrDelegate() throws Exception { - /* we not want a mock of resourcemgr deleagte */ + /* we not want a mock of resource mgr delegate */ final ClientRMProtocol clientRMProtocol = mock(ClientRMProtocol.class); ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) { @Override @@ -256,6 +262,9 @@ public synchronized void start() { @Test public void testHistoryServerToken() throws Exception { + //Set the master principal in the config + conf.set(YarnConfiguration.RM_PRINCIPAL,"foo@LOCAL"); + final String masterPrincipal = Master.getMasterPrincipal(conf); final MRClientProtocol hsProxy = mock(MRClientProtocol.class); @@ -265,7 +274,7 @@ public GetDelegationTokenResponse answer(InvocationOnMock invocation) { GetDelegationTokenRequest request = (GetDelegationTokenRequest)invocation.getArguments()[0]; // check that the renewer matches the cluster's RM principal - assertEquals(request.getRenewer(), masterPrincipal); + assertEquals(masterPrincipal, request.getRenewer() ); DelegationToken token = recordFactory.newRecordInstance(DelegationToken.class); @@ -357,4 +366,53 @@ public void testAMAdminCommandOpts() throws Exception { assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex); } } + @Test + public void testWarnCommandOpts() throws Exception { + Logger logger = Logger.getLogger(YARNRunner.class); + + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + Layout layout = new SimpleLayout(); + Appender appender = new WriterAppender(layout, bout); + logger.addAppender(appender); + + JobConf jobConf = new JobConf(); + + jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true -Djava.library.path=foo"); + jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m -Djava.library.path=bar"); + + YARNRunner yarnRunner = new YARNRunner(jobConf); + + File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE); + OutputStream out = new FileOutputStream(jobxml); + conf.writeXml(out); + out.close(); + + File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT); + out = new FileOutputStream(jobsplit); + out.close(); + + File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO); + out = new FileOutputStream(jobsplitmetainfo); + out.close(); + + File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE); + out = new FileOutputStream(appTokens); + out.close(); + + @SuppressWarnings("unused") + ApplicationSubmissionContext submissionContext = + yarnRunner.createApplicationSubmissionContext(jobConf, testWorkDir.toString(), new Credentials()); + + String logMsg = bout.toString(); + assertTrue(logMsg.contains("WARN - Usage of -Djava.library.path in " + + "yarn.app.mapreduce.am.admin-command-opts can cause programs to no " + + "longer function if hadoop native libraries are used. These values " + + "should be set as part of the LD_LIBRARY_PATH in the app master JVM " + + "env using yarn.app.mapreduce.am.admin.user.env config settings.")); + assertTrue(logMsg.contains("WARN - Usage of -Djava.library.path in " + + "yarn.app.mapreduce.am.command-opts can cause programs to no longer " + + "function if hadoop native libraries are used. These values should " + + "be set as part of the LD_LIBRARY_PATH in the app master JVM env " + + "using yarn.app.mapreduce.am.env config settings.")); + } }