big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5113. Streaming input/output types are ignored with java mapper/reducer. (sandyr via tucu)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1463307 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2013-04-01 21:42:12 +00:00
parent 7c65dbb938
commit 806073867e
4 changed files with 29 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-mapreduce-project/CHANGES.txt
View File

@ -158,6 +158,21 @@ Trunk (Unreleased)
MAPREDUCE-5006. Fix failing streaming tests due to MAPREDUCE-4994. MAPREDUCE-5006. Fix failing streaming tests due to MAPREDUCE-4994.
(Sandy Ryza via tomwhite) (Sandy Ryza via tomwhite)
Release 2.0.5-alpha - UNRELEASED
INCOMPATIBLE CHANGES
NEW FEATURES
IMPROVEMENTS
OPTIMIZATIONS
BUG FIXES
MAPREDUCE-5113. Streaming input/output types are ignored with java
mapper/reducer. (sandyr via tucu)
Release 2.0.4-beta - UNRELEASED Release 2.0.4-beta - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

4
hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
View File

@ -875,7 +875,7 @@ protected void setJobConf() throws IOException {
IdentifierResolver.TEXT_ID)); IdentifierResolver.TEXT_ID));
jobConf_.setClass("stream.map.output.reader.class", jobConf_.setClass("stream.map.output.reader.class",
idResolver.getOutputReaderClass(), OutputReader.class); idResolver.getOutputReaderClass(), OutputReader.class);
if (isMapperACommand) { if (isMapperACommand || jobConf_.get("stream.map.output") != null) {
// if mapper is a command, then map output key/value classes come from the // if mapper is a command, then map output key/value classes come from the
// idResolver // idResolver
jobConf_.setMapOutputKeyClass(idResolver.getOutputKeyClass()); jobConf_.setMapOutputKeyClass(idResolver.getOutputKeyClass());
@ -891,7 +891,7 @@ protected void setJobConf() throws IOException {
IdentifierResolver.TEXT_ID)); IdentifierResolver.TEXT_ID));
jobConf_.setClass("stream.reduce.output.reader.class", jobConf_.setClass("stream.reduce.output.reader.class",
idResolver.getOutputReaderClass(), OutputReader.class); idResolver.getOutputReaderClass(), OutputReader.class);
if (isReducerACommand) { if (isReducerACommand || jobConf_.get("stream.reduce.output") != null) {
// if reducer is a command, then output key/value classes come from the // if reducer is a command, then output key/value classes come from the
// idResolver // idResolver
jobConf_.setOutputKeyClass(idResolver.getOutputKeyClass()); jobConf_.setOutputKeyClass(idResolver.getOutputKeyClass());

12
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.MRJobConfig;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@ -178,6 +179,17 @@ public void testCommandMapperAndCommandReducerAndZeroReduces()
super.testCommandLine(); super.testCommandLine();
} }
@Test
public void testDefaultToIdentityReducer() throws Exception {
args.add("-mapper");args.add(map);
args.add("-jobconf");
args.add("mapreduce.task.files.preserve.failedtasks=true");
args.add("-jobconf");
args.add("stream.tmpdir="+System.getProperty("test.build.data","/tmp"));
args.add("-inputformat");args.add(TextInputFormat.class.getName());
super.testCommandLine();
}
@Override @Override
@Test @Test
public void testCommandLine() { public void testCommandLine() {

1
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
View File

@ -43,7 +43,6 @@ void testParentJobConfToEnvVars() throws IOException
// Note the dots translated to underscore: // Note the dots translated to underscore:
// property names have been escaped in PipeMapRed.safeEnvVarName() // property names have been escaped in PipeMapRed.safeEnvVarName()
expectDefined("mapreduce_cluster_local_dir"); expectDefined("mapreduce_cluster_local_dir");
expect("mapred_output_format_class", "org.apache.hadoop.mapred.TextOutputFormat");
expect("mapreduce_map_output_key_class", "org.apache.hadoop.io.Text"); expect("mapreduce_map_output_key_class", "org.apache.hadoop.io.Text");
expect("mapreduce_map_output_value_class", "org.apache.hadoop.io.Text"); expect("mapreduce_map_output_value_class", "org.apache.hadoop.io.Text");