big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3158. Fix test failures in MRv1 due to default framework being set to yarn. Contributed by Hitesh Shah.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1181310 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2011-10-11 00:26:41 +00:00
parent a8190ce5c5
commit 42e93829e5
16 changed files with 44 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -1561,6 +1561,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-3154. Fix JobSubmitter to check for output specs before copying
job submission files to fail fast. (Abhijit Suresh Shingate via acmurthy)
MAPREDUCE-3158. Fix test failures in MRv1 due to default framework being
set to yarn. (Hitesh Shah)
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
View File

@ -314,6 +314,8 @@ private void runSubtask(org.apache.hadoop.mapred.Task task,
ReduceTask reduce = (ReduceTask)task;
// a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner:
// set framework name to local to make task local
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
conf.set(MRConfig.MASTER_ADDRESS, "local"); // bypass shuffle
reduce.run(conf, umbilical);

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
View File

@ -342,11 +342,9 @@ public void run(JobConf job, final TaskUmbilicalProtocol umbilical)
RawKeyValueIterator rIter = null;
boolean isLocal = false;
// local iff framework == classic && master address == local
// local iff framework == local
String framework = job.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
isLocal = "local".equals(job.get(MRConfig.MASTER_ADDRESS, "local"));
}
isLocal = framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME);
if (!isLocal) {
Class combinerClass = conf.getCombinerClass();

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
View File

@ -67,6 +67,7 @@ public interface MRConfig {
public static final String FRAMEWORK_NAME = "mapreduce.framework.name";
public static final String CLASSIC_FRAMEWORK_NAME = "classic";
public static final String YARN_FRAMEWORK_NAME = "yarn";
public static final String LOCAL_FRAMEWORK_NAME = "local";
public static final String TASK_LOCAL_OUTPUT_CLASS =
"mapreduce.task.local.output.class";

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
View File

@ -135,7 +135,7 @@ public void testSleepJob() throws IOException, InterruptedException,
}
Configuration sleepConf = new Configuration(mrCluster.getConfig());
// set master address to local to test that local mode applied iff framework == classic and master_address == local
// set master address to local to test that local mode applied iff framework == local
sleepConf.set(MRConfig.MASTER_ADDRESS, "local");
SleepJob sleepJob = new SleepJob();

2
hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java
View File

@ -34,7 +34,7 @@ public class LocalClientProtocolProvider extends ClientProtocolProvider {
@Override
public ClientProtocol create(Configuration conf) throws IOException {
String framework = conf.get(MRConfig.FRAMEWORK_NAME);
if (framework != null && !framework.equals("local")) {
if (framework != null && !framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME)) {
return null;
}
String tracker = conf.get(JTConfig.JT_IPC_ADDRESS, "local");

10
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.MRConfig;
import java.io.File;
import java.io.IOException;
@ -200,7 +201,14 @@ protected FileSystem getFileSystem() {
* @return configuration that works on the testcase Hadoop instance
*/
protected JobConf createJobConf() {
return (localMR) ? new JobConf() : mrCluster.createJobConf();
if (localMR) {
JobConf conf = new JobConf();
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
return conf;
}
else {
return mrCluster.createJobConf();
}
}
}

2
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCollect.java
View File

@ -20,6 +20,7 @@
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.UtilsForTests.RandomInputFormat;
import org.apache.hadoop.mapreduce.MRConfig;
import junit.framework.TestCase;
import java.io.*;
@ -120,6 +121,7 @@ public void configure(JobConf conf) throws IOException {
conf.setOutputValueClass(IntWritable.class);
FileOutputFormat.setOutputPath(conf, OUTPUT_DIR);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
conf.setMapperClass(Map.class);
conf.setReducerClass(Reduce.class);
conf.setNumMapTasks(1);

3
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestComparators.java
View File

@ -19,6 +19,8 @@
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.MRConfig;
import junit.framework.TestCase;
import java.io.*;
import java.util.*;
@ -305,6 +307,7 @@ public void configure() throws Exception {
conf.setMapOutputValueClass(IntWritable.class);
// set up two map jobs, so we can test merge phase in Reduce also
conf.setNumMapTasks(2);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
conf.setOutputFormat(SequenceFileOutputFormat.class);
if (!fs.mkdirs(testdir)) {

3
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java
View File

@ -23,6 +23,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.MRConfig;
import java.io.DataOutputStream;
import java.io.IOException;
@ -77,6 +78,8 @@ public void testCustomFile() throws Exception {
conf.setMapperClass(TestMap.class);
conf.setReducerClass(TestReduce.class);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
FileInputFormat.setInputPaths(conf, inDir);
FileOutputFormat.setOutputPath(conf, outDir);

5
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java
View File

@ -36,6 +36,7 @@
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.serializer.JavaSerializationComparator;
import org.apache.hadoop.mapreduce.MRConfig;
public class TestJavaSerialization extends TestCase {
@ -109,6 +110,8 @@ public void testMapReduceJob() throws Exception {
conf.setMapperClass(WordCountMapper.class);
conf.setReducerClass(SumReducer.class);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
FileInputFormat.setInputPaths(conf, INPUT_DIR);
FileOutputFormat.setOutputPath(conf, OUTPUT_DIR);
@ -155,6 +158,8 @@ public void testWriteToSequencefile() throws Exception {
conf.setMapperClass(WordCountMapper.class);
conf.setReducerClass(SumReducer.class);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
FileInputFormat.setInputPaths(conf, INPUT_DIR);
FileOutputFormat.setOutputPath(conf, OUTPUT_DIR);

3
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapOutputType.java
View File

@ -20,6 +20,7 @@
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.lib.*;
import org.apache.hadoop.mapreduce.MRConfig;
import junit.framework.TestCase;
import java.io.*;
import java.util.*;
@ -90,6 +91,8 @@ public void configure() throws Exception {
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(Text.class);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
conf.setOutputFormat(SequenceFileOutputFormat.class);
if (!fs.mkdirs(testdir)) {
throw new IOException("Mkdirs failed to create " + testdir.toString());

5
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java
View File

@ -43,6 +43,7 @@
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.mapred.lib.IdentityMapper;
import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
@ -351,6 +352,7 @@ public void testNullKeys() throws Exception {
conf.setInputFormat(SequenceFileInputFormat.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
conf.setNumReduceTasks(1);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
JobClient.runJob(conf);
@ -382,6 +384,7 @@ private void checkCompression(boolean compressMapOutputs,
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(Text.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
if (includeCombine) {
conf.setCombinerClass(IdentityReducer.class);
}
@ -445,6 +448,7 @@ public void launch() throws Exception {
} else {
conf = new JobConf(getConf());
}
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
conf.setJarByClass(TestMapRed.class);
int countsToGo = counts;
int dist[] = new int[range];
@ -737,6 +741,7 @@ public void runJob(int items) {
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(Text.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
if (!fs.mkdirs(testdir)) {
throw new IOException("Mkdirs failed to create " + testdir.toString());
}

1
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java
View File

@ -30,6 +30,7 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.MRConfig;
import java.io.DataInput;
import java.io.DataOutput;

2
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java
View File

@ -43,7 +43,7 @@ public void testClusterWithLocalClientProvider() throws Exception {
}
try {
conf.set(MRConfig.FRAMEWORK_NAME, "local");
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
conf.set(JTConfig.JT_IPC_ADDRESS, "127.0.0.1:0");
new Cluster(conf);

2
hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapCollection.java
View File

@ -40,6 +40,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.util.ReflectionUtils;
public class TestMapCollection {
@ -311,6 +312,7 @@ private static void runTest(String name, int keylen, int vallen,
private static void runTest(String name, Job job) throws Exception {
job.setNumReduceTasks(1);
job.getConfiguration().set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
job.getConfiguration().setInt(MRJobConfig.IO_SORT_FACTOR, 1000);
job.getConfiguration().set("fs.default.name", "file:///");
job.getConfiguration().setInt("test.mapcollection.num.maps", 1);