YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. Contributed by Craig Welch.
This commit is contained in:
parent
3164e7d838
commit
4102e5882e
@ -931,10 +931,16 @@ static ContainerLaunchContext createContainerLaunchContext(
|
|||||||
// Fill in the fields needed per-container that are missing in the common
|
// Fill in the fields needed per-container that are missing in the common
|
||||||
// spec.
|
// spec.
|
||||||
|
|
||||||
|
boolean userClassesTakesPrecedence =
|
||||||
|
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
|
||||||
|
|
||||||
// Setup environment by cloning from common env.
|
// Setup environment by cloning from common env.
|
||||||
Map<String, String> env = commonContainerSpec.getEnvironment();
|
Map<String, String> env = commonContainerSpec.getEnvironment();
|
||||||
Map<String, String> myEnv = new HashMap<String, String>(env.size());
|
Map<String, String> myEnv = new HashMap<String, String>(env.size());
|
||||||
myEnv.putAll(env);
|
myEnv.putAll(env);
|
||||||
|
if (userClassesTakesPrecedence) {
|
||||||
|
myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
|
||||||
|
}
|
||||||
MapReduceChildJVM.setVMEnv(myEnv, remoteTask);
|
MapReduceChildJVM.setVMEnv(myEnv, remoteTask);
|
||||||
|
|
||||||
// Set up the launch command
|
// Set up the launch command
|
||||||
|
@ -241,11 +241,6 @@ public static void setClasspath(Map<String, String> environment,
|
|||||||
boolean userClassesTakesPrecedence =
|
boolean userClassesTakesPrecedence =
|
||||||
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
|
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
|
||||||
|
|
||||||
if (userClassesTakesPrecedence) {
|
|
||||||
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
|
|
||||||
"true");
|
|
||||||
}
|
|
||||||
|
|
||||||
String classpathEnvVar =
|
String classpathEnvVar =
|
||||||
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
|
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
|
||||||
? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();
|
? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();
|
||||||
|
@ -19,6 +19,7 @@
|
|||||||
package org.apache.hadoop.yarn.api;
|
package org.apache.hadoop.yarn.api;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
@ -190,6 +191,13 @@ public enum Environment {
|
|||||||
*/
|
*/
|
||||||
HADOOP_YARN_HOME("HADOOP_YARN_HOME"),
|
HADOOP_YARN_HOME("HADOOP_YARN_HOME"),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* $CLASSPATH_PREPEND_DISTCACHE
|
||||||
|
* Private, Windows specific
|
||||||
|
*/
|
||||||
|
@Private
|
||||||
|
CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"),
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* $CONTAINER_ID
|
* $CONTAINER_ID
|
||||||
* Final, exported by NodeManager and non-modifiable by users.
|
* Final, exported by NodeManager and non-modifiable by users.
|
||||||
|
@ -611,6 +611,7 @@ private static void addDeprecatedKeys() {
|
|||||||
ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
|
ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
|
||||||
ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
|
ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
|
||||||
ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
|
ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
|
||||||
|
ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(),
|
||||||
ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
|
ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
|
||||||
|
|
||||||
/** address of node manager IPC.*/
|
/** address of node manager IPC.*/
|
||||||
@ -1279,16 +1280,6 @@ private static void addDeprecatedKeys() {
|
|||||||
public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
|
public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
|
||||||
+ "application.classpath";
|
+ "application.classpath";
|
||||||
|
|
||||||
/**
|
|
||||||
* Whether or not entries from the distributed cache should be preferred over
|
|
||||||
* the rest of the YARN CLASSPATH
|
|
||||||
*/
|
|
||||||
public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE =
|
|
||||||
YARN_PREFIX + "application.classpath.prepend.distcache";
|
|
||||||
|
|
||||||
public static final boolean
|
|
||||||
DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Default platform-agnostic CLASSPATH for YARN applications. A
|
* Default platform-agnostic CLASSPATH for YARN applications. A
|
||||||
* comma-separated list of CLASSPATH entries. The parameter expansion marker
|
* comma-separated list of CLASSPATH entries. The parameter expansion marker
|
||||||
|
@ -749,11 +749,12 @@ public void sanitizeEnv(Map<String, String> environment, Path pwd,
|
|||||||
//jar is created and so they "are lost" and have to be explicitly
|
//jar is created and so they "are lost" and have to be explicitly
|
||||||
//added to the classpath instead. This also means that their position
|
//added to the classpath instead. This also means that their position
|
||||||
//is lost relative to other non-distcache classpath entries which will
|
//is lost relative to other non-distcache classpath entries which will
|
||||||
//break things like mapreduce.job.user.classpath.first.
|
//break things like mapreduce.job.user.classpath.first. An environment
|
||||||
|
//variable can be set to indicate that distcache entries should come
|
||||||
|
//first
|
||||||
|
|
||||||
boolean preferLocalizedJars = conf.getBoolean(
|
boolean preferLocalizedJars = Boolean.valueOf(
|
||||||
YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
|
environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
|
||||||
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE
|
|
||||||
);
|
);
|
||||||
|
|
||||||
boolean needsSeparator = false;
|
boolean needsSeparator = false;
|
||||||
|
@ -417,7 +417,7 @@ public void testPrependDistcache() throws Exception {
|
|||||||
userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
|
userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
|
||||||
userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
|
userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
|
||||||
userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
|
userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
|
||||||
userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH");
|
userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
|
||||||
containerLaunchContext.setEnvironment(userSetEnv);
|
containerLaunchContext.setEnvironment(userSetEnv);
|
||||||
Container container = mock(Container.class);
|
Container container = mock(Container.class);
|
||||||
when(container.getContainerId()).thenReturn(cId);
|
when(container.getContainerId()).thenReturn(cId);
|
||||||
@ -463,14 +463,12 @@ public void handle(Event event) {
|
|||||||
Assert.assertTrue(
|
Assert.assertTrue(
|
||||||
result.get(result.size() - 1).endsWith("userjarlink.jar"));
|
result.get(result.size() - 1).endsWith("userjarlink.jar"));
|
||||||
|
|
||||||
//Now move userjar to the front
|
//Then, with user classpath first
|
||||||
|
userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
|
||||||
|
|
||||||
cId = ContainerId.newContainerId(appAttemptId, 1);
|
cId = ContainerId.newContainerId(appAttemptId, 1);
|
||||||
when(container.getContainerId()).thenReturn(cId);
|
when(container.getContainerId()).thenReturn(cId);
|
||||||
|
|
||||||
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
|
|
||||||
"true");
|
|
||||||
|
|
||||||
launch = new ContainerLaunch(distContext, conf,
|
launch = new ContainerLaunch(distContext, conf,
|
||||||
dispatcher, exec, null, container, dirsHandler, containerManager);
|
dispatcher, exec, null, container, dirsHandler, containerManager);
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user