From bb81a17e0b4fd7aa78604a6bb018ada8996f5674 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 22 Jan 2013 16:26:29 +0000 Subject: [PATCH] MAPREDUCE-4949. Enable multiple pi jobs to run in parallel. (sandyr via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1437029 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../hadoop/examples/QuasiMonteCarlo.java | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c6e75770bd..f4c6ce947c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -209,6 +209,8 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4907. TrackerDistributedCacheManager issues too many getFileStatus calls. (sandyr via tucu) + MAPREDUCE-4949. Enable multiple pi jobs to run in parallel. (sandyr via tucu) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java index 08d7cfad2d..8ddd64fd1f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.RoundingMode; +import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -77,8 +78,7 @@ public class QuasiMonteCarlo extends Configured implements Tool { static final String DESCRIPTION = "A map/reduce program that estimates Pi using a quasi-Monte Carlo method."; /** tmp directory for input/output */ - static private final Path TMP_DIR = new Path( - QuasiMonteCarlo.class.getSimpleName() + "_TMP_3_141592654"); + static private final String TMP_DIR_PREFIX = QuasiMonteCarlo.class.getSimpleName(); /** 2-dimensional Halton sequence {H(i)}, * where H(i) is a 2-dimensional point and i >= 1 is the index. @@ -228,9 +228,9 @@ public void reduce(BooleanWritable isInside, @Override public void cleanup(Context context) throws IOException { //write output to a file - Path outDir = new Path(TMP_DIR, "out"); - Path outFile = new Path(outDir, "reduce-out"); Configuration conf = context.getConfiguration(); + Path outDir = new Path(conf.get(FileOutputFormat.OUTDIR)); + Path outFile = new Path(outDir, "reduce-out"); FileSystem fileSys = FileSystem.get(conf); SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, outFile, LongWritable.class, LongWritable.class, @@ -246,7 +246,7 @@ public void cleanup(Context context) throws IOException { * @return the estimated value of Pi */ public static BigDecimal estimatePi(int numMaps, long numPoints, - Configuration conf + Path tmpDir, Configuration conf ) throws IOException, ClassNotFoundException, InterruptedException { Job job = new Job(conf); //setup job conf @@ -269,14 +269,14 @@ public static BigDecimal estimatePi(int numMaps, long numPoints, job.setSpeculativeExecution(false); //setup input/output directories - final Path inDir = new Path(TMP_DIR, "in"); - final Path outDir = new Path(TMP_DIR, "out"); + final Path inDir = new Path(tmpDir, "in"); + final Path outDir = new Path(tmpDir, "out"); FileInputFormat.setInputPaths(job, inDir); FileOutputFormat.setOutputPath(job, outDir); final FileSystem fs = FileSystem.get(conf); - if (fs.exists(TMP_DIR)) { - throw new IOException("Tmp directory " + fs.makeQualified(TMP_DIR) + if (fs.exists(tmpDir)) { + throw new IOException("Tmp directory " + fs.makeQualified(tmpDir) + " already exists. Please remove it first."); } if (!fs.mkdirs(inDir)) { @@ -325,7 +325,7 @@ public static BigDecimal estimatePi(int numMaps, long numPoints, .multiply(BigDecimal.valueOf(numInside.get())) .divide(numTotal, RoundingMode.HALF_UP); } finally { - fs.delete(TMP_DIR, true); + fs.delete(tmpDir, true); } } @@ -344,12 +344,15 @@ public int run(String[] args) throws Exception { final int nMaps = Integer.parseInt(args[0]); final long nSamples = Long.parseLong(args[1]); + long now = System.currentTimeMillis(); + int rand = new Random().nextInt(Integer.MAX_VALUE); + final Path tmpDir = new Path(TMP_DIR_PREFIX + "_" + now + "_" + rand); System.out.println("Number of Maps = " + nMaps); System.out.println("Samples per Map = " + nSamples); System.out.println("Estimated value of Pi is " - + estimatePi(nMaps, nSamples, getConf())); + + estimatePi(nMaps, nSamples, tmpDir, getConf())); return 0; }