big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6577. MR AM unable to load native library without MR_AM_ADMIN_USER_ENV set (sjlee)

Browse Source
This commit is contained in:
Sangjin Lee 2016-01-05 15:22:50 -08:00
parent dec8fedb65
commit f6f16118d3
6 changed files with 94 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -315,6 +315,9 @@ Release 2.9.0 - UNRELEASED
BUG FIXES
MAPREDUCE-6577. MR AM unable to load native library without
MR_AM_ADMIN_USER_ENV set (sjlee)
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES

21
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -740,6 +740,16 @@ public interface MRJobConfig {
public static final String MR_AM_ADMIN_USER_ENV =
MR_AM_PREFIX + "admin.user.env";
// although the AM admin user env default should be the same as the task user
// env default, there are problems in making it work on Windows currently
// MAPREDUCE-6588 should address the issue and set it to a proper non-empty
// value
public static final String DEFAULT_MR_AM_ADMIN_USER_ENV =
Shell.WINDOWS ?
"" :
"LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
"/lib/native";
public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile";
public static final boolean DEFAULT_MR_AM_PROFILE = false;
public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX
@ -763,10 +773,13 @@ public interface MRJobConfig {
public static final String MAPRED_ADMIN_USER_ENV =
"mapreduce.admin.user.env";
public final String DEFAULT_MAPRED_ADMIN_USER_ENV =
Shell.WINDOWS ?
"PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin":
"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native";
// the "%...%" macros can be expanded prematurely and are probably not OK
// this should be addressed by MAPREDUCE-6588
public static final String DEFAULT_MAPRED_ADMIN_USER_ENV =
Shell.WINDOWS ?
"PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin" :
"LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
"/lib/native";
public static final String WORKDIR = "work";

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
View File

@ -472,13 +472,14 @@ public ApplicationSubmissionContext createApplicationSubmissionContext(
conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL,
MRJobConfig.DEFAULT_SHELL));
// Add the container working directory at the front of LD_LIBRARY_PATH
// Add the container working directory in front of LD_LIBRARY_PATH
MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
// Setup the environment variables for Admin first
MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), conf);
conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV,
MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV), conf);
// Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ENV), conf);

66
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
View File

@ -395,56 +395,56 @@ public static void tearDown() {
/**
* To test OS dependent setting of default execution path for a MapRed task.
* Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set -
* for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH - for
* Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
* for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH -
* for Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
* LD_LIBRARY_PATH
*/
@Test
public void testMapRedExecutionEnv() {
// test if the env variable can be set
try {
// Application environment
Map<String, String> environment = new HashMap<String, String>();
String setupHadoopHomeCommand = Shell.WINDOWS ?
"HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home" :
"HADOOP_COMMON_HOME=/fake/path/to/hadoop/common/home";
MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
// for windows, test if the env variable can be set
// this may be removed as part of MAPREDUCE-6588
if (Shell.WINDOWS) {
try {
// Application environment
Map<String, String> environment = new HashMap<String, String>();
String setupHadoopHomeCommand =
"HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home";
MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
// Add the env variables passed by the admin
MRApps.setEnvFromInputString(environment, conf.get(
MRJobConfig.MAPRED_ADMIN_USER_ENV,
MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
String executionPaths = environment.get(
Shell.WINDOWS ? "PATH" : "LD_LIBRARY_PATH");
String toFind = Shell.WINDOWS ?
"C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin" :
"/fake/path/to/hadoop/common/home/lib/native";
// Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
assertTrue("execution path does not include the hadoop lib location "
+ toFind, executionPaths.contains(toFind));
} catch (Exception e) {
e.printStackTrace();
fail("Exception in testing execution environment for MapReduce task");
tearDown();
// Add the env variables passed by the admin
MRApps.setEnvFromInputString(environment, conf.get(
MRJobConfig.MAPRED_ADMIN_USER_ENV,
MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
String executionPaths = environment.get("PATH");
String toFind =
"C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin";
// Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
assertTrue("execution path does not include the hadoop lib location "
+ toFind, executionPaths.contains(toFind));
} catch (Exception e) {
e.printStackTrace();
fail("Exception in testing execution environment for MapReduce task");
tearDown();
}
}
// now launch a mapreduce job to ensure that the child
// also gets the configured setting for hadoop lib
try {
JobConf conf = new JobConf(mr.getConfig());
JobConf conf = new JobConf(mr.getConfig());
// initialize input, output directories
Path inDir = new Path("input");
Path outDir = new Path("output");
String input = "The input";
// set config to use the ExecutionEnvCheckMapClass map class
configure(conf, inDir, outDir, input,
ExecutionEnvCheckMapClass.class, IdentityReducer.class);
launchTest(conf, inDir, outDir, input);
} catch(Exception e) {
e.printStackTrace();
fail("Exception in testing propagation of env setting to child task");

47
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
View File

@ -63,6 +63,7 @@
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@ -570,16 +571,34 @@ public void testNodeLabelExp() throws Exception {
}
@Test
public void testAMStandardEnv() throws Exception {
public void testAMStandardEnvWithDefaultLibPath() throws Exception {
testAMStandardEnv(false);
}
@Test
public void testAMStandardEnvWithCustomLibPath() throws Exception {
testAMStandardEnv(true);
}
private void testAMStandardEnv(boolean customLibPath) throws Exception {
// the Windows behavior is different and this test currently doesn't really
// apply
// MAPREDUCE-6588 should revisit this test
if (Shell.WINDOWS) {
return;
}
final String ADMIN_LIB_PATH = "foo";
final String USER_LIB_PATH = "bar";
final String USER_SHELL = "shell";
JobConf jobConf = new JobConf();
String pathKey = Environment.LD_LIBRARY_PATH.name();
jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
ADMIN_LIB_PATH);
jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
+ USER_LIB_PATH);
if (customLibPath) {
jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, pathKey + "=" +
ADMIN_LIB_PATH);
jobConf.set(MRJobConfig.MR_AM_ENV, pathKey + "=" + USER_LIB_PATH);
}
jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);
YARNRunner yarnRunner = new YARNRunner(jobConf);
@ -589,15 +608,23 @@ public void testAMStandardEnv() throws Exception {
// make sure PWD is first in the lib path
ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
Map<String, String> env = clc.getEnvironment();
String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
assertNotNull("LD_LIBRARY_PATH not set", libPath);
String libPath = env.get(pathKey);
assertNotNull(pathKey + " not set", libPath);
String cps = jobConf.getBoolean(
MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
assertEquals("Bad AM LD_LIBRARY_PATH setting",
MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
+ cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);
String expectedLibPath =
MRApps.crossPlatformifyMREnv(conf, Environment.PWD);
if (customLibPath) {
// append admin libpath and user libpath
expectedLibPath += cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH;
} else {
expectedLibPath += cps +
MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV.substring(
pathKey.length() + 1);
}
assertEquals("Bad AM " + pathKey + " setting", expectedLibPath, libPath);
// make sure SHELL is set
String shell = env.get(Environment.SHELL.name());

1
hadoop-project/pom.xml
View File

@ -1133,6 +1133,7 @@
<forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
<argLine>${maven-surefire-plugin.argLine}</argLine>
<environmentVariables>
<HADOOP_COMMON_HOME>${hadoop.common.build.dir}</HADOOP_COMMON_HOME>
<!-- HADOOP_HOME required for tests on Windows to find winutils -->
<HADOOP_HOME>${hadoop.common.build.dir}</HADOOP_HOME>
<LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib</LD_LIBRARY_PATH>