big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3727. jobtoken location property in jobconf refers to wrong jobtoken file (tucu)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1240410 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2012-02-03 23:48:48 +00:00
parent cf897b6f3c
commit b030231108
4 changed files with 32 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -670,6 +670,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3708. Metrics: Incorrect Apps Submitted Count (Bhallamudi via
mahadev)
MAPREDUCE-3727. jobtoken location property in jobconf refers to wrong
jobtoken file (tucu)
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
View File

@ -369,6 +369,12 @@ JobStatus submitJobInternal(Job job, Cluster cluster)
conf.set(toFullPropertyName(queue,
QueueACL.ADMINISTER_JOBS.getAclName()), acl.getAclString());
// removing jobtoken referrals before copying the jobconf to HDFS
// as the tasks don't need this setting, actually they may break
// because of it if present as the referral will point to a
// different job.
TokenCache.cleanUpTokenReferral(conf);
// Write job file to submit dir
writeConf(conf, submitJobFile);

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
View File

@ -79,7 +79,17 @@ public static void obtainTokensForNamenodes(Credentials credentials,
}
obtainTokensForNamenodesInternal(credentials, ps, conf);
}
/**
* Remove jobtoken referrals which don't make sense in the context
* of the task execution.
*
* @param conf
*/
public static void cleanUpTokenReferral(Configuration conf) {
conf.unset(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY);
}
static void obtainTokensForNamenodesInternal(Credentials credentials,
Path[] ps, Configuration conf) throws IOException {
for(Path p: ps) {

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
View File

@ -19,6 +19,8 @@
package org.apache.hadoop.mapreduce.security;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@ -33,6 +35,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Master;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -158,4 +161,13 @@ public List<Token<?>> answer(InvocationOnMock invocation)
return mockFs;
}
@Test
public void testCleanUpTokenReferral() throws Exception {
Configuration conf = new Configuration();
conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, "foo");
TokenCache.cleanUpTokenReferral(conf);
assertNull(conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY));
}
}