big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5083. MiniMRCluster should use a random component when creating an actual cluster. Contributed by Siddharth Seth.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1459932 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Hitesh Shah 2013-03-22 19:11:01 +00:00
parent 6f92ad9868
commit b8c74a8a25
3 changed files with 16 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -250,6 +250,9 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-3872. Fix an event handling races in ContainerLauncherImpl. MAPREDUCE-3872. Fix an event handling races in ContainerLauncherImpl.
(Robert Kanter via sseth) (Robert Kanter via sseth)
MAPREDUCE-5083. MiniMRCluster should use a random component when creating an
actual cluster (Siddharth Seth via hitesh)
Release 2.0.4-alpha - UNRELEASED Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

13
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java
View File

@ -38,6 +38,11 @@ public class MiniMRClientClusterFactory {
public static MiniMRClientCluster create(Class<?> caller, int noOfNMs, public static MiniMRClientCluster create(Class<?> caller, int noOfNMs,
Configuration conf) throws IOException { Configuration conf) throws IOException {
return create(caller, caller.getSimpleName(), noOfNMs, conf);
}
public static MiniMRClientCluster create(Class<?> caller, String identifier,
int noOfNMs, Configuration conf) throws IOException {
if (conf == null) { if (conf == null) {
conf = new Configuration(); conf = new Configuration();
@ -45,7 +50,7 @@ public static MiniMRClientCluster create(Class<?> caller, int noOfNMs,
FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(conf);
Path testRootDir = new Path("target", caller.getSimpleName() + "-tmpDir") Path testRootDir = new Path("target", identifier + "-tmpDir")
.makeQualified(fs); .makeQualified(fs);
Path appJar = new Path(testRootDir, "MRAppJar.jar"); Path appJar = new Path(testRootDir, "MRAppJar.jar");
@ -65,10 +70,10 @@ public static MiniMRClientCluster create(Class<?> caller, int noOfNMs,
fs.setPermission(remoteCallerJar, new FsPermission("744")); fs.setPermission(remoteCallerJar, new FsPermission("744"));
job.addFileToClassPath(remoteCallerJar); job.addFileToClassPath(remoteCallerJar);
MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier,
.getSimpleName(), noOfNMs); noOfNMs);
job.getConfiguration().set("minimrclientcluster.caller.name", job.getConfiguration().set("minimrclientcluster.caller.name",
caller.getSimpleName()); identifier);
job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number", job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number",
noOfNMs); noOfNMs);
miniMRYarnCluster.init(job.getConfiguration()); miniMRYarnCluster.init(job.getConfiguration());

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.mapred; package org.apache.hadoop.mapred;
import java.io.IOException; import java.io.IOException;
import java.util.Random;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -177,8 +178,10 @@ public MiniMRCluster(int jobTrackerPort, int taskTrackerPort,
int numTrackerToExclude, Clock clock) throws IOException { int numTrackerToExclude, Clock clock) throws IOException {
if (conf == null) conf = new JobConf(); if (conf == null) conf = new JobConf();
FileSystem.setDefaultUri(conf, namenode); FileSystem.setDefaultUri(conf, namenode);
String identifier = this.getClass().getSimpleName() + "_"
+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE));
mrClientCluster = MiniMRClientClusterFactory.create(this.getClass(), mrClientCluster = MiniMRClientClusterFactory.create(this.getClass(),
numTaskTrackers, conf); identifier, numTaskTrackers, conf);
} }
public UserGroupInformation getUgi() { public UserGroupInformation getUgi() {