diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d04ce5daed..ce22824d8d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -315,6 +315,9 @@ Release 2.9.0 - UNRELEASED BUG FIXES + MAPREDUCE-6577. MR AM unable to load native library without + MR_AM_ADMIN_USER_ENV set (sjlee) + Release 2.8.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 3d1e841bea..c98746ae2f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -740,6 +740,16 @@ public interface MRJobConfig { public static final String MR_AM_ADMIN_USER_ENV = MR_AM_PREFIX + "admin.user.env"; + // although the AM admin user env default should be the same as the task user + // env default, there are problems in making it work on Windows currently + // MAPREDUCE-6588 should address the issue and set it to a proper non-empty + // value + public static final String DEFAULT_MR_AM_ADMIN_USER_ENV = + Shell.WINDOWS ? + "" : + "LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") + + "/lib/native"; + public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile"; public static final boolean DEFAULT_MR_AM_PROFILE = false; public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX @@ -763,10 +773,13 @@ public interface MRJobConfig { public static final String MAPRED_ADMIN_USER_ENV = "mapreduce.admin.user.env"; - public final String DEFAULT_MAPRED_ADMIN_USER_ENV = - Shell.WINDOWS ? - "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin": - "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native"; + // the "%...%" macros can be expanded prematurely and are probably not OK + // this should be addressed by MAPREDUCE-6588 + public static final String DEFAULT_MAPRED_ADMIN_USER_ENV = + Shell.WINDOWS ? + "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin" : + "LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") + + "/lib/native"; public static final String WORKDIR = "work"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index f15b5c1979..fe631de99a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -472,13 +472,14 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL)); - // Add the container working directory at the front of LD_LIBRARY_PATH + // Add the container working directory in front of LD_LIBRARY_PATH MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(), MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf); // Setup the environment variables for Admin first MRApps.setEnvFromInputString(environment, - conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), conf); + conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV, + MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV), conf); // Setup the environment variables (LD_LIBRARY_PATH, etc) MRApps.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ENV), conf); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java index 59aa00f4d2..74be8a758d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java @@ -395,56 +395,56 @@ public static void tearDown() { /** * To test OS dependent setting of default execution path for a MapRed task. * Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set - - * for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH - for - * Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in + * for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH - + * for Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in * LD_LIBRARY_PATH */ @Test public void testMapRedExecutionEnv() { - // test if the env variable can be set - try { - // Application environment - Map environment = new HashMap(); - String setupHadoopHomeCommand = Shell.WINDOWS ? - "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home" : - "HADOOP_COMMON_HOME=/fake/path/to/hadoop/common/home"; - MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf); + // for windows, test if the env variable can be set + // this may be removed as part of MAPREDUCE-6588 + if (Shell.WINDOWS) { + try { + // Application environment + Map environment = new HashMap(); + String setupHadoopHomeCommand = + "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home"; + MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf); - // Add the env variables passed by the admin - MRApps.setEnvFromInputString(environment, conf.get( - MRJobConfig.MAPRED_ADMIN_USER_ENV, - MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf); - - String executionPaths = environment.get( - Shell.WINDOWS ? "PATH" : "LD_LIBRARY_PATH"); - String toFind = Shell.WINDOWS ? - "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin" : - "/fake/path/to/hadoop/common/home/lib/native"; - - // Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib - assertTrue("execution path does not include the hadoop lib location " - + toFind, executionPaths.contains(toFind)); - } catch (Exception e) { - e.printStackTrace(); - fail("Exception in testing execution environment for MapReduce task"); - tearDown(); + // Add the env variables passed by the admin + MRApps.setEnvFromInputString(environment, conf.get( + MRJobConfig.MAPRED_ADMIN_USER_ENV, + MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf); + + String executionPaths = environment.get("PATH"); + String toFind = + "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin"; + + // Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib + assertTrue("execution path does not include the hadoop lib location " + + toFind, executionPaths.contains(toFind)); + } catch (Exception e) { + e.printStackTrace(); + fail("Exception in testing execution environment for MapReduce task"); + tearDown(); + } } - + // now launch a mapreduce job to ensure that the child // also gets the configured setting for hadoop lib try { - - JobConf conf = new JobConf(mr.getConfig()); + + JobConf conf = new JobConf(mr.getConfig()); // initialize input, output directories Path inDir = new Path("input"); Path outDir = new Path("output"); String input = "The input"; - + // set config to use the ExecutionEnvCheckMapClass map class configure(conf, inDir, outDir, input, ExecutionEnvCheckMapClass.class, IdentityReducer.class); launchTest(conf, inDir, outDir, input); - + } catch(Exception e) { e.printStackTrace(); fail("Exception in testing propagation of env setting to child task"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 3293a9d515..0e34042821 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -63,6 +63,7 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; @@ -570,16 +571,34 @@ public void testNodeLabelExp() throws Exception { } @Test - public void testAMStandardEnv() throws Exception { + public void testAMStandardEnvWithDefaultLibPath() throws Exception { + testAMStandardEnv(false); + } + + @Test + public void testAMStandardEnvWithCustomLibPath() throws Exception { + testAMStandardEnv(true); + } + + private void testAMStandardEnv(boolean customLibPath) throws Exception { + // the Windows behavior is different and this test currently doesn't really + // apply + // MAPREDUCE-6588 should revisit this test + if (Shell.WINDOWS) { + return; + } + final String ADMIN_LIB_PATH = "foo"; final String USER_LIB_PATH = "bar"; final String USER_SHELL = "shell"; JobConf jobConf = new JobConf(); + String pathKey = Environment.LD_LIBRARY_PATH.name(); - jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" + - ADMIN_LIB_PATH); - jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH=" - + USER_LIB_PATH); + if (customLibPath) { + jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, pathKey + "=" + + ADMIN_LIB_PATH); + jobConf.set(MRJobConfig.MR_AM_ENV, pathKey + "=" + USER_LIB_PATH); + } jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL); YARNRunner yarnRunner = new YARNRunner(jobConf); @@ -589,15 +608,23 @@ public void testAMStandardEnv() throws Exception { // make sure PWD is first in the lib path ContainerLaunchContext clc = appSubCtx.getAMContainerSpec(); Map env = clc.getEnvironment(); - String libPath = env.get(Environment.LD_LIBRARY_PATH.name()); - assertNotNull("LD_LIBRARY_PATH not set", libPath); + String libPath = env.get(pathKey); + assertNotNull(pathKey + " not set", libPath); String cps = jobConf.getBoolean( MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM) ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator; - assertEquals("Bad AM LD_LIBRARY_PATH setting", - MRApps.crossPlatformifyMREnv(conf, Environment.PWD) - + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath); + String expectedLibPath = + MRApps.crossPlatformifyMREnv(conf, Environment.PWD); + if (customLibPath) { + // append admin libpath and user libpath + expectedLibPath += cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH; + } else { + expectedLibPath += cps + + MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV.substring( + pathKey.length() + 1); + } + assertEquals("Bad AM " + pathKey + " setting", expectedLibPath, libPath); // make sure SHELL is set String shell = env.get(Environment.SHELL.name()); diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index bb844f419f..36d735ad95 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1133,6 +1133,7 @@ 900 ${maven-surefire-plugin.argLine} + ${hadoop.common.build.dir} ${hadoop.common.build.dir} ${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib