big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1138. yarn.application.classpath is set to point to etc., which does not work on Windows. Contributed by Chuan Liu.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1556978 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2014-01-09 23:04:18 +00:00
parent b4d6c5823b
commit 8b2c8a657e
4 changed files with 39 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
View File

@ -182,9 +182,10 @@ public void testSetClasspath() throws IOException {
MRApps.setClasspath(environment, job.getConfiguration()); MRApps.setClasspath(environment, job.getConfiguration());
assertTrue(environment.get("CLASSPATH").startsWith( assertTrue(environment.get("CLASSPATH").startsWith(
ApplicationConstants.Environment.PWD.$() + File.pathSeparator)); ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
String yarnAppClasspath = String yarnAppClasspath = job.getConfiguration().get(
job.getConfiguration().get( YarnConfiguration.YARN_APPLICATION_CLASSPATH,
YarnConfiguration.YARN_APPLICATION_CLASSPATH); StringUtils.join(",",
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
if (yarnAppClasspath != null) { if (yarnAppClasspath != null) {
yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", File.pathSeparator) yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", File.pathSeparator)
.trim(); .trim();
@ -217,7 +218,10 @@ public void testSetClasspathWithArchives () throws IOException {
MRApps.setClasspath(environment, conf); MRApps.setClasspath(environment, conf);
assertTrue(environment.get("CLASSPATH").startsWith( assertTrue(environment.get("CLASSPATH").startsWith(
ApplicationConstants.Environment.PWD.$() + File.pathSeparator)); ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH); String confClasspath = job.getConfiguration().get(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
StringUtils.join(",",
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
if (confClasspath != null) { if (confClasspath != null) {
confClasspath = confClasspath.replaceAll(",\\s*", File.pathSeparator) confClasspath = confClasspath.replaceAll(",\\s*", File.pathSeparator)
.trim(); .trim();

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
View File

@ -31,6 +31,7 @@
import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
@ -86,8 +87,10 @@ private void startCluster(Configuration conf) throws Exception {
conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.block.access.token.enable", "false");
conf.set("dfs.permissions", "true"); conf.set("dfs.permissions", "true");
conf.set("hadoop.security.authentication", "simple"); conf.set("hadoop.security.authentication", "simple");
String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) + String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
File.pathSeparator + classpathDir; StringUtils.join(",",
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH))
+ File.pathSeparator + classpathDir;
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp); conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp);
dfsCluster = new MiniDFSCluster(conf, 1, true, null); dfsCluster = new MiniDFSCluster(conf, 1, true, null);
FileSystem fileSystem = dfsCluster.getFileSystem(); FileSystem fileSystem = dfsCluster.getFileSystem();

3
hadoop-yarn-project/CHANGES.txt
View File

@ -395,6 +395,9 @@ Release 2.3.0 - UNRELEASED
YARN-1438. Ensure container diagnostics includes exception from container YARN-1438. Ensure container diagnostics includes exception from container
launch. (stevel via acmurthy) launch. (stevel via acmurthy)
YARN-1138. yarn.application.classpath is set to point to $HADOOP_CONF_DIR
etc., which does not work on Windows. (Chuan Liu via cnauroth)
Release 2.2.0 - 2013-10-13 Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1016,10 +1016,29 @@
<!-- Applications' Configuration--> <!-- Applications' Configuration-->
<property> <property>
<description>CLASSPATH for YARN applications. A comma-separated list <description>
of CLASSPATH entries</description> CLASSPATH for YARN applications. A comma-separated list
<name>yarn.application.classpath</name> of CLASSPATH entries. When this value is empty, the following default
<value>$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/share/hadoop/common/*,$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,$HADOOP_YARN_HOME/share/hadoop/yarn/*,$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*</value> CLASSPATH for YARN applications would be used.
For Linux:
$HADOOP_CONF_DIR,
$HADOOP_COMMON_HOME/share/hadoop/common/*,
$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,
$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,
$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,
$HADOOP_YARN_HOME/share/hadoop/yarn/*,
$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*
For Windows:
%HADOOP_CONF_DIR%,
%HADOOP_COMMON_HOME%/share/hadoop/common/*,
%HADOOP_COMMON_HOME%/share/hadoop/common/lib/*,
%HADOOP_HDFS_HOME%/share/hadoop/hdfs/*,
%HADOOP_HDFS_HOME%/share/hadoop/hdfs/lib/*,
%HADOOP_YARN_HOME%/share/hadoop/yarn/*,
%HADOOP_YARN_HOME%/share/hadoop/yarn/lib/*
</description>
<name>yarn.application.classpath</name>
<value></value>
</property> </property>
<!-- Other configuration --> <!-- Other configuration -->