diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 28627d9286..5e341a1909 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -392,6 +392,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3490. Fixed MapReduce AM to count failed maps also towards Reduce ramp up. (Sharad Agarwal and Arun C Murthy via vinodkv) + MAPREDUCE-1744. DistributedCache creates its own FileSytem instance when + adding a file/archive to the path. (Dick King via tucu) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java index e4351536c8..88de3f88c1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java @@ -1030,7 +1030,7 @@ public void addCacheFile(URI uri) { public void addFileToClassPath(Path file) throws IOException { ensureState(JobState.DEFINE); - DistributedCache.addFileToClassPath(file, conf); + DistributedCache.addFileToClassPath(file, conf, file.getFileSystem(conf)); } /** @@ -1045,7 +1045,7 @@ public void addFileToClassPath(Path file) public void addArchiveToClassPath(Path archive) throws IOException { ensureState(JobState.DEFINE); - DistributedCache.addArchiveToClassPath(archive, conf); + DistributedCache.addArchiveToClassPath(archive, conf, archive.getFileSystem(conf)); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java index 8b1d3a61f4..4040358142 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java @@ -269,7 +269,7 @@ public static void addCacheFile(URI uri, Configuration conf) { /** * Add an file path to the current set of classpath entries It adds the file * to cache as well. Intended to be used by user code. - * + * * @param file Path of the file to be added * @param conf Configuration that contains the classpath setting * @deprecated Use {@link Job#addFileToClassPath(Path)} instead @@ -277,12 +277,25 @@ public static void addCacheFile(URI uri, Configuration conf) { @Deprecated public static void addFileToClassPath(Path file, Configuration conf) throws IOException { + addFileToClassPath(file, conf, file.getFileSystem(conf)); + } + + /** + * Add a file path to the current set of classpath entries. It adds the file + * to cache as well. Intended to be used by user code. + * + * @param file Path of the file to be added + * @param conf Configuration that contains the classpath setting + * @param fs FileSystem with respect to which {@code archivefile} should + * be interpreted. + */ + public static void addFileToClassPath + (Path file, Configuration conf, FileSystem fs) + throws IOException { String classpath = conf.get(MRJobConfig.CLASSPATH_FILES); conf.set(MRJobConfig.CLASSPATH_FILES, classpath == null ? file.toString() : classpath + "," + file.toString()); - FileSystem fs = FileSystem.get(conf); URI uri = fs.makeQualified(file).toUri(); - addCacheFile(uri, conf); } @@ -318,10 +331,23 @@ public static Path[] getFileClassPaths(Configuration conf) { @Deprecated public static void addArchiveToClassPath(Path archive, Configuration conf) throws IOException { + addArchiveToClassPath(archive, conf, archive.getFileSystem(conf)); + } + + /** + * Add an archive path to the current set of classpath entries. It adds the + * archive to cache as well. Intended to be used by user code. + * + * @param archive Path of the archive to be added + * @param conf Configuration that contains the classpath setting + * @param fs FileSystem with respect to which {@code archive} should be interpreted. + */ + public static void addArchiveToClassPath + (Path archive, Configuration conf, FileSystem fs) + throws IOException { String classpath = conf.get(MRJobConfig.CLASSPATH_ARCHIVES); conf.set(MRJobConfig.CLASSPATH_ARCHIVES, classpath == null ? archive .toString() : classpath + "," + archive.toString()); - FileSystem fs = FileSystem.get(conf); URI uri = fs.makeQualified(archive).toUri(); addCacheArchive(uri, conf); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java index 79a14fc40e..7af592297b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java @@ -196,7 +196,7 @@ static boolean runJob(JobConf conf, Path inDir, Path outDir, int numMaps, file.close(); } - DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf); + DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf, fs); conf.setOutputCommitter(CustomOutputCommitter.class); conf.setInputFormat(TextInputFormat.class); conf.setOutputKeyClass(LongWritable.class);