big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6022. map_input_file is missing from streaming job

Browse Source 
environment. Contributed by Jason Lowe.
This commit is contained in:
Kihwal Lee 2014-10-29 12:28:32 -05:00
parent 782971ae7a
commit b056048114
4 changed files with 35 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -558,6 +558,32 @@ public static boolean isDeprecated(String key) {
return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
}
/**
* Sets all deprecated properties that are not currently set but have a
* corresponding new property that is set. Useful for iterating the
* properties when all deprecated properties for currently set properties
* need to be present.
*/
public void setDeprecatedProperties() {
DeprecationContext deprecations = deprecationContext.get();
Properties props = getProps();
Properties overlay = getOverlay();
for (Map.Entry<String, DeprecatedKeyInfo> entry :
deprecations.getDeprecatedKeyMap().entrySet()) {
String depKey = entry.getKey();
if (!overlay.contains(depKey)) {
for (String newKey : entry.getValue().newKeys) {
String val = overlay.getProperty(newKey);
if (val != null) {
props.setProperty(depKey, val);
overlay.setProperty(depKey, val);
break;
}
}
}
}
}
/**
* Checks for the presence of the property <code>name</code> in the
* deprecation map. Returns the first of the list of new keys if present

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -450,6 +450,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6142. Fixed test failures in TestJobHistoryEventHandler and
TestMRTimelineEventHandling. (Zhijie Shen via vinodkv)
MAPREDUCE-6022. map_input_file is missing from streaming job environment.
(jlowe via kihwal)
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES

4
hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
View File

@ -235,7 +235,9 @@ void setStreamJobDetails(JobConf job) {
}
}
void addJobConfToEnvironment(JobConf conf, Properties env) {
void addJobConfToEnvironment(JobConf jobconf, Properties env) {
JobConf conf = new JobConf(jobconf);
conf.setDeprecatedProperties();
Iterator it = conf.iterator();
while (it.hasNext()) {
Map.Entry en = (Map.Entry) it.next();

3
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
View File

@ -57,6 +57,9 @@ void testParentJobConfToEnvVars() throws IOException
// the FileSplit context properties are not available in local hadoop..
// so can't check them in this test.
// verify some deprecated properties appear for older stream jobs
expect("map_input_file", env.getProperty("mapreduce_map_input_file"));
expect("map_input_length", env.getProperty("mapreduce_map_input_length"));
}
// this runs in a subprocess; won't use JUnit's assertTrue()