big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3604. Fixed streaming to use new mapreduce.framework.name to check for local mode.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1225259 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2011-12-28 18:23:12 +00:00
parent 1e0c51c66e
commit 919f56c3d4
3 changed files with 62 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -369,6 +369,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3586. Modified CompositeService to avoid duplicate stop operations MAPREDUCE-3586. Modified CompositeService to avoid duplicate stop operations
thereby solving race conditions in MR AM shutdown. (vinodkv) thereby solving race conditions in MR AM shutdown. (vinodkv)
MAPREDUCE-3604. Fixed streaming to use new mapreduce.framework.name to
check for local mode. (acmurthy)
Release 0.23.0 - 2011-11-01 Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

17
hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java
View File

@ -18,16 +18,19 @@
package org.apache.hadoop.streaming; package org.apache.hadoop.streaming;
import java.io.*; import java.io.File;
import java.net.*; import java.io.FileInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URL;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.mapreduce.MRConfig;
/** /**
* Utilities used in streaming * Utilities used in streaming
@ -189,6 +192,8 @@ static Environment env() {
} }
public static boolean isLocalJobTracker(JobConf job) { public static boolean isLocalJobTracker(JobConf job) {
return job.get(JTConfig.JT_IPC_ADDRESS, "local").equals("local"); String framework =
job.get(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
return framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME);
} }
} }

48
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMRFramework.java Normal file
View File

@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.streaming;
import static org.junit.Assert.*;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.junit.Test;
public class TestMRFramework {
@Test
public void testFramework() {
JobConf jobConf = new JobConf();
jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
assertFalse("Expected 'isLocal' to be false",
StreamUtil.isLocalJobTracker(jobConf));
jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
assertFalse("Expected 'isLocal' to be false",
StreamUtil.isLocalJobTracker(jobConf));
jobConf.set(JTConfig.JT_IPC_ADDRESS, "jthost:9090");
jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
assertTrue("Expected 'isLocal' to be true",
StreamUtil.isLocalJobTracker(jobConf));
}
}