big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7036. ASF License warning in hadoop-mapreduce-client

Browse Source 
Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
This commit is contained in:
Takanobu Asanuma 2018-01-30 00:36:33 +09:00 committed by Akira Ajisaka
parent e9c72d04be
commit 56feaa40bb
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
2 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
View File

@ -45,6 +45,7 @@
import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TaskAttemptID; import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
@ -103,8 +104,8 @@ class Application<K1 extends WritableComparable, V1 extends Writable,
// This password is used as shared secret key between this application and // This password is used as shared secret key between this application and
// child pipes process // child pipes process
byte[] password = jobToken.getPassword(); byte[] password = jobToken.getPassword();
String localPasswordFile = new File(".") + Path.SEPARATOR String localPasswordFile = new File(conf.get(MRConfig.LOCAL_DIR))
+ "jobTokenPassword"; + Path.SEPARATOR + "jobTokenPassword";
writePasswordToLocalFile(localPasswordFile, password, conf); writePasswordToLocalFile(localPasswordFile, password, conf);
env.put("hadoop.pipes.shared.secret.location", localPasswordFile); env.put("hadoop.pipes.shared.secret.location", localPasswordFile);

19
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
View File

@ -47,6 +47,7 @@
import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.IFile.Writer; import org.apache.hadoop.mapred.IFile.Writer;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.Counters;
@ -83,10 +84,10 @@ public class TestPipeApplication {
public void testRunner() throws Exception { public void testRunner() throws Exception {
// clean old password files // clean old password files
File[] psw = cleanTokenPasswordFile(); JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf);
try { try {
RecordReader<FloatWritable, NullWritable> rReader = new ReaderPipesMapRunner(); RecordReader<FloatWritable, NullWritable> rReader = new ReaderPipesMapRunner();
JobConf conf = new JobConf();
conf.set(Submitter.IS_JAVA_RR, "true"); conf.set(Submitter.IS_JAVA_RR, "true");
// for stdour and stderror // for stdour and stderror
@ -162,7 +163,7 @@ public void testApplication() throws Throwable {
TestTaskReporter reporter = new TestTaskReporter(); TestTaskReporter reporter = new TestTaskReporter();
File[] psw = cleanTokenPasswordFile(); File[] psw = cleanTokenPasswordFile(conf);
try { try {
conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName); conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName);
@ -247,7 +248,7 @@ public void testSubmitter() throws Exception {
JobConf conf = new JobConf(); JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(); File[] psw = cleanTokenPasswordFile(conf);
System.setProperty("test.build.data", System.setProperty("test.build.data",
"target/tmp/build/TEST_SUBMITTER_MAPPER/data"); "target/tmp/build/TEST_SUBMITTER_MAPPER/data");
@ -388,8 +389,8 @@ public void testSubmitter() throws Exception {
@Test @Test
public void testPipesReduser() throws Exception { public void testPipesReduser() throws Exception {
File[] psw = cleanTokenPasswordFile();
JobConf conf = new JobConf(); JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf);
try { try {
Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>( Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>(
"user".getBytes(), "password".getBytes(), new Text("kind"), new Text( "user".getBytes(), "password".getBytes(), new Text("kind"), new Text(
@ -506,14 +507,16 @@ public void progress() {
} }
private File[] cleanTokenPasswordFile() throws Exception { private File[] cleanTokenPasswordFile(JobConf conf) throws Exception {
File[] result = new File[2]; File[] result = new File[2];
result[0] = new File("./jobTokenPassword"); result[0] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
+ "jobTokenPassword");
if (result[0].exists()) { if (result[0].exists()) {
FileUtil.chmod(result[0].getAbsolutePath(), "700"); FileUtil.chmod(result[0].getAbsolutePath(), "700");
assertTrue(result[0].delete()); assertTrue(result[0].delete());
} }
result[1] = new File("./.jobTokenPassword.crc"); result[1] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
+ ".jobTokenPassword.crc");
if (result[1].exists()) { if (result[1].exists()) {
FileUtil.chmod(result[1].getAbsolutePath(), "700"); FileUtil.chmod(result[1].getAbsolutePath(), "700");
result[1].delete(); result[1].delete();