big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda)

Browse Source 
Change-Id: Ie8f01425d58409fa3661f768205b7616128c8aa4
(cherry picked from commit 035e0f97ea44b0495707949a781d8792dcf6ea6b)
This commit is contained in:
Wangda Tan 2018-04-12 14:33:33 -07:00
parent 9031a76d44
commit 995cba65fe
2 changed files with 10 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
View File

@ -45,7 +45,6 @@
import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TaskAttemptID; import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
@ -104,8 +103,8 @@ class Application<K1 extends WritableComparable, V1 extends Writable,
// This password is used as shared secret key between this application and // This password is used as shared secret key between this application and
// child pipes process // child pipes process
byte[] password = jobToken.getPassword(); byte[] password = jobToken.getPassword();
String localPasswordFile = new File(conf.get(MRConfig.LOCAL_DIR)) String localPasswordFile = new File(".") + Path.SEPARATOR
+ Path.SEPARATOR + "jobTokenPassword"; + "jobTokenPassword";
writePasswordToLocalFile(localPasswordFile, password, conf); writePasswordToLocalFile(localPasswordFile, password, conf);
env.put("hadoop.pipes.shared.secret.location", localPasswordFile); env.put("hadoop.pipes.shared.secret.location", localPasswordFile);

19
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
View File

@ -47,7 +47,6 @@
import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.IFile.Writer; import org.apache.hadoop.mapred.IFile.Writer;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.Counters;
@ -84,10 +83,10 @@ public class TestPipeApplication {
public void testRunner() throws Exception { public void testRunner() throws Exception {
// clean old password files // clean old password files
JobConf conf = new JobConf(); File[] psw = cleanTokenPasswordFile();
File[] psw = cleanTokenPasswordFile(conf);
try { try {
RecordReader<FloatWritable, NullWritable> rReader = new ReaderPipesMapRunner(); RecordReader<FloatWritable, NullWritable> rReader = new ReaderPipesMapRunner();
JobConf conf = new JobConf();
conf.set(Submitter.IS_JAVA_RR, "true"); conf.set(Submitter.IS_JAVA_RR, "true");
// for stdour and stderror // for stdour and stderror
@ -163,7 +162,7 @@ public void testApplication() throws Throwable {
TestTaskReporter reporter = new TestTaskReporter(); TestTaskReporter reporter = new TestTaskReporter();
File[] psw = cleanTokenPasswordFile(conf); File[] psw = cleanTokenPasswordFile();
try { try {
conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName); conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName);
@ -248,7 +247,7 @@ public void testSubmitter() throws Exception {
JobConf conf = new JobConf(); JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf); File[] psw = cleanTokenPasswordFile();
System.setProperty("test.build.data", System.setProperty("test.build.data",
"target/tmp/build/TEST_SUBMITTER_MAPPER/data"); "target/tmp/build/TEST_SUBMITTER_MAPPER/data");
@ -389,8 +388,8 @@ public void testSubmitter() throws Exception {
@Test @Test
public void testPipesReduser() throws Exception { public void testPipesReduser() throws Exception {
File[] psw = cleanTokenPasswordFile();
JobConf conf = new JobConf(); JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf);
try { try {
Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>( Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>(
"user".getBytes(), "password".getBytes(), new Text("kind"), new Text( "user".getBytes(), "password".getBytes(), new Text("kind"), new Text(
@ -507,16 +506,14 @@ public void progress() {
} }
private File[] cleanTokenPasswordFile(JobConf conf) throws Exception { private File[] cleanTokenPasswordFile() throws Exception {
File[] result = new File[2]; File[] result = new File[2];
result[0] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR result[0] = new File("./jobTokenPassword");
+ "jobTokenPassword");
if (result[0].exists()) { if (result[0].exists()) {
FileUtil.chmod(result[0].getAbsolutePath(), "700"); FileUtil.chmod(result[0].getAbsolutePath(), "700");
assertTrue(result[0].delete()); assertTrue(result[0].delete());
} }
result[1] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR result[1] = new File("./.jobTokenPassword.crc");
+ ".jobTokenPassword.crc");
if (result[1].exists()) { if (result[1].exists()) {
FileUtil.chmod(result[1].getAbsolutePath(), "700"); FileUtil.chmod(result[1].getAbsolutePath(), "700");
result[1].delete(); result[1].delete();