diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 7e82df2a9b..3055a25e12 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -931,10 +931,16 @@ static ContainerLaunchContext createContainerLaunchContext( // Fill in the fields needed per-container that are missing in the common // spec. + boolean userClassesTakesPrecedence = + conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); + // Setup environment by cloning from common env. Map env = commonContainerSpec.getEnvironment(); Map myEnv = new HashMap(env.size()); myEnv.putAll(env); + if (userClassesTakesPrecedence) { + myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true"); + } MapReduceChildJVM.setVMEnv(myEnv, remoteTask); // Set up the launch command diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 6d82104d6a..c8d8a44555 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -241,11 +241,6 @@ public static void setClasspath(Map environment, boolean userClassesTakesPrecedence = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); - if (userClassesTakesPrecedence) { - conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - "true"); - } - String classpathEnvVar = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false) ? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index d42cd67d85..42464da1a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; @@ -190,6 +191,13 @@ public enum Environment { */ HADOOP_YARN_HOME("HADOOP_YARN_HOME"), + /** + * $CLASSPATH_PREPEND_DISTCACHE + * Private, Windows specific + */ + @Private + CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"), + /** * $CONTAINER_ID * Final, exported by NodeManager and non-modifiable by users. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index cc9b44ac10..1b6d9c9ea3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -611,6 +611,7 @@ private static void addDeprecatedKeys() { ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(), ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(), ApplicationConstants.Environment.HADOOP_CONF_DIR.key(), + ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(), ApplicationConstants.Environment.HADOOP_YARN_HOME.key())); /** address of node manager IPC.*/ @@ -1279,16 +1280,6 @@ private static void addDeprecatedKeys() { public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX + "application.classpath"; - /** - * Whether or not entries from the distributed cache should be preferred over - * the rest of the YARN CLASSPATH - */ - public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = - YARN_PREFIX + "application.classpath.prepend.distcache"; - - public static final boolean - DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false; - /** * Default platform-agnostic CLASSPATH for YARN applications. A * comma-separated list of CLASSPATH entries. The parameter expansion marker diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 0b7f762fd6..af168c5943 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -749,11 +749,12 @@ public void sanitizeEnv(Map environment, Path pwd, //jar is created and so they "are lost" and have to be explicitly //added to the classpath instead. This also means that their position //is lost relative to other non-distcache classpath entries which will - //break things like mapreduce.job.user.classpath.first. + //break things like mapreduce.job.user.classpath.first. An environment + //variable can be set to indicate that distcache entries should come + //first - boolean preferLocalizedJars = conf.getBoolean( - YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE + boolean preferLocalizedJars = Boolean.valueOf( + environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name()) ); boolean needsSeparator = false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index efeeb1dcce..285635771f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -417,7 +417,7 @@ public void testPrependDistcache() throws Exception { userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME"); userSetEnv.put(Environment.PWD.name(), "user_set_PWD"); userSetEnv.put(Environment.HOME.name(), "user_set_HOME"); - userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH"); + userSetEnv.put(Environment.CLASSPATH.name(), "APATH"); containerLaunchContext.setEnvironment(userSetEnv); Container container = mock(Container.class); when(container.getContainerId()).thenReturn(cId); @@ -463,14 +463,12 @@ public void handle(Event event) { Assert.assertTrue( result.get(result.size() - 1).endsWith("userjarlink.jar")); - //Now move userjar to the front + //Then, with user classpath first + userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true"); cId = ContainerId.newContainerId(appAttemptId, 1); when(container.getContainerId()).thenReturn(cId); - conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - "true"); - launch = new ContainerLaunch(distContext, conf, dispatcher, exec, null, container, dirsHandler, containerManager);