big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda)

Browse Source 
Change-Id: Ie8f01425d58409fa3661f768205b7616128c8aa4
(cherry picked from commit 035e0f97ea44b0495707949a781d8792dcf6ea6b)
This commit is contained in:
Wangda Tan 2018-04-12 14:33:33 -07:00
parent 9031a76d44
commit 995cba65fe
2 changed files with 10 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
View File

@ -45,7 +45,6 @@
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
@ -104,8 +103,8 @@ class Application<K1 extends WritableComparable, V1 extends Writable,
// This password is used as shared secret key between this application and
// child pipes process
byte[] password = jobToken.getPassword();
String localPasswordFile = new File(conf.get(MRConfig.LOCAL_DIR))
+ Path.SEPARATOR + "jobTokenPassword";
String localPasswordFile = new File(".") + Path.SEPARATOR
+ "jobTokenPassword";
writePasswordToLocalFile(localPasswordFile, password, conf);
env.put("hadoop.pipes.shared.secret.location", localPasswordFile);

19
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
View File

@ -47,7 +47,6 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.IFile.Writer;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapred.Counters;
@ -84,10 +83,10 @@ public class TestPipeApplication {
public void testRunner() throws Exception {
// clean old password files
JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf);
File[] psw = cleanTokenPasswordFile();
try {
RecordReader<FloatWritable, NullWritable> rReader = new ReaderPipesMapRunner();
JobConf conf = new JobConf();
conf.set(Submitter.IS_JAVA_RR, "true");
// for stdour and stderror
@ -163,7 +162,7 @@ public void testApplication() throws Throwable {
TestTaskReporter reporter = new TestTaskReporter();
File[] psw = cleanTokenPasswordFile(conf);
File[] psw = cleanTokenPasswordFile();
try {
conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName);
@ -248,7 +247,7 @@ public void testSubmitter() throws Exception {
JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf);
File[] psw = cleanTokenPasswordFile();
System.setProperty("test.build.data",
"target/tmp/build/TEST_SUBMITTER_MAPPER/data");
@ -389,8 +388,8 @@ public void testSubmitter() throws Exception {
@Test
public void testPipesReduser() throws Exception {
File[] psw = cleanTokenPasswordFile();
JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf);
try {
Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>(
"user".getBytes(), "password".getBytes(), new Text("kind"), new Text(
@ -507,16 +506,14 @@ public void progress() {
}
private File[] cleanTokenPasswordFile(JobConf conf) throws Exception {
private File[] cleanTokenPasswordFile() throws Exception {
File[] result = new File[2];
result[0] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
+ "jobTokenPassword");
result[0] = new File("./jobTokenPassword");
if (result[0].exists()) {
FileUtil.chmod(result[0].getAbsolutePath(), "700");
assertTrue(result[0].delete());
}
result[1] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
+ ".jobTokenPassword.crc");
result[1] = new File("./.jobTokenPassword.crc");
if (result[1].exists()) {
FileUtil.chmod(result[1].getAbsolutePath(), "700");
result[1].delete();