big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-4638. MR AM supplies MapReduce jars in classpath rather than rely on YARN. Contributed by Arun C Murthy.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1381868 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2012-09-07 04:56:32 +00:00
parent e8c193bf24
commit 0056914773
5 changed files with 44 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -249,6 +249,9 @@ Release 2.1.0-alpha - Unreleased
MAPREDUCE-4579. Split TestTaskAttempt into two so as to pass tests on MAPREDUCE-4579. Split TestTaskAttempt into two so as to pass tests on
jdk7. (Thomas Graves via vinodkv) jdk7. (Thomas Graves via vinodkv)
MAPREDUCE-4638. MR AM supplies MapReduce jars in classpath rather than
rely on YARN. (acmurthy)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
View File

@ -179,6 +179,12 @@ private static void setMRFrameworkClasspath(
Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
.trim()); .trim());
} }
for (String c : conf.getStrings(
MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
.trim());
}
} finally { } finally {
if (classpathFileStream != null) { if (classpathFileStream != null) {
classpathFileStream.close(); classpathFileStream.close();

16
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
View File

@ -140,11 +140,19 @@ public class TestMRApps {
Map<String, String> environment = new HashMap<String, String>(); Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, job.getConfiguration()); MRApps.setClasspath(environment, job.getConfiguration());
assertTrue(environment.get("CLASSPATH").startsWith("$PWD:")); assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH); String yarnAppClasspath =
if (confClasspath != null) { job.getConfiguration().get(
confClasspath = confClasspath.replaceAll(",\\s*", ":").trim(); YarnConfiguration.YARN_APPLICATION_CLASSPATH);
if (yarnAppClasspath != null) {
yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", ":").trim();
} }
assertTrue(environment.get("CLASSPATH").contains(confClasspath)); assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
String mrAppClasspath =
job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH);
if (mrAppClasspath != null) {
mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", ":").trim();
}
assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
} }
@Test public void testSetClasspathWithUserPrecendence() { @Test public void testSetClasspathWithUserPrecendence() {

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -587,4 +587,18 @@ public interface MRJobConfig {
MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT = MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT =
"security.job.client.protocol.acl"; "security.job.client.protocol.acl";
/**
* CLASSPATH for all YARN MapReduce applications.
*/
public static final String MAPREDUCE_APPLICATION_CLASSPATH =
"mapreduce.application.classpath";
/**
* Default CLASSPATH for all YARN MapReduce applications.
*/
public static final String[] DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = {
"$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*",
"$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*",
};
} }

9
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
View File

@ -1203,6 +1203,8 @@
<!-- end of node health script variables --> <!-- end of node health script variables -->
<!-- MR YARN Application properties -->
<property> <property>
<name>mapreduce.job.counters.limit</name> <name>mapreduce.job.counters.limit</name>
<value>120</value> <value>120</value>
@ -1312,6 +1314,13 @@
<description>The amount of memory the MR AppMaster needs.</description> <description>The amount of memory the MR AppMaster needs.</description>
</property> </property>
<property>
<description>CLASSPATH for MR applications. A comma-separated list
of CLASSPATH entries</description>
<name>mapreduce.application.classpath</name>
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
</property>
<!-- jobhistory properties --> <!-- jobhistory properties -->
<property> <property>