big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5699. Allow setting tags on MR jobs (kasha)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565384 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Karthik Kambatla 2014-02-06 18:29:30 +00:00
parent ab96a0838d
commit 4594b74b85
4 changed files with 20 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-mapreduce-project/CHANGES.txt
View File

@ -153,6 +153,8 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5732. Report proper queue when job has been automatically placed MAPREDUCE-5732. Report proper queue when job has been automatically placed
(Sandy Ryza) (Sandy Ryza)
MAPREDUCE-5699. Allow setting tags on MR jobs (kasha)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -60,6 +60,8 @@ public interface MRJobConfig {
public static final String QUEUE_NAME = "mapreduce.job.queuename"; public static final String QUEUE_NAME = "mapreduce.job.queuename";
public static final String JOB_TAGS = "mapreduce.job.tags";
public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks"; public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks";
public static final String SPLIT_FILE = "mapreduce.job.splitfile"; public static final String SPLIT_FILE = "mapreduce.job.splitfile";

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
View File

@ -1,4 +1,5 @@
<?xml version="1.0"?> <?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- <!--
Licensed to the Apache Software Foundation (ASF) under one or more Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with contributor license agreements. See the NOTICE file distributed with
@ -15,7 +16,6 @@
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
--> -->
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Do not modify this file directly. Instead, copy entries that you --> <!-- Do not modify this file directly. Instead, copy entries that you -->
<!-- wish to modify from this file into mapred-site.xml and change them --> <!-- wish to modify from this file into mapred-site.xml and change them -->
@ -727,6 +727,14 @@
</description> </description>
</property> </property>
<property>
<name>mapreduce.job.tags</name>
<value></value>
<description> Tags for the job that will be passed to YARN at submission
time. Queries to YARN for applications can filter on these tags.
</description>
</property>
<property> <property>
<name>mapreduce.cluster.local.dir</name> <name>mapreduce.cluster.local.dir</name>
<value>${hadoop.tmp.dir}/mapred/local</value> <value>${hadoop.tmp.dir}/mapred/local</value>

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
View File

@ -21,7 +21,9 @@
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Vector; import java.util.Vector;
@ -467,6 +469,8 @@ public ApplicationSubmissionContext createApplicationSubmissionContext(
ContainerLaunchContext.newInstance(localResources, environment, ContainerLaunchContext.newInstance(localResources, environment,
vargsFinal, null, securityTokens, acls); vargsFinal, null, securityTokens, acls);
Collection<String> tagsFromConf =
jobConf.getTrimmedStringCollection(MRJobConfig.JOB_TAGS);
// Set up the ApplicationSubmissionContext // Set up the ApplicationSubmissionContext
ApplicationSubmissionContext appContext = ApplicationSubmissionContext appContext =
@ -486,6 +490,9 @@ public ApplicationSubmissionContext createApplicationSubmissionContext(
MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS)); MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));
appContext.setResource(capability); appContext.setResource(capability);
appContext.setApplicationType(MRJobConfig.MR_APPLICATION_TYPE); appContext.setApplicationType(MRJobConfig.MR_APPLICATION_TYPE);
if (tagsFromConf != null && !tagsFromConf.isEmpty()) {
appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
}
return appContext; return appContext;
} }