From 78331cfecacdf12639ef4c4023418ddf875e87bf Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Mon, 3 Jun 2013 23:42:28 +0000 Subject: [PATCH] MAPREDUCE-5300. Fix backward incompatibility for o.a.h.mapreduce.filecache.DistributedCache. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1489227 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../hadoop/mapreduce/v2/util/MRApps.java | 15 ++-------- .../mapred/TestMRWithDistributedCache.java | 4 +-- .../mapreduce/filecache/DistributedCache.java | 30 +++++++++++++++---- .../hadoop/mapreduce/task/JobContextImpl.java | 22 ++++++++++++-- 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index feaa754c29..7531bdd49b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -487,6 +487,9 @@ Release 2.1.0-beta - UNRELEASED MAPREDUCE-5296. Fix backward incompatibility for JobControl by adding the omitted addJob. (Zhijie Shen via acmurthy) + MAPREDUCE-5300. Fix backward incompatibility for + o.a.h.mapreduce.filecache.DistributedCache. (Zhijie Shen via acmurthy) + BREAKDOWN OF HADOOP-8562 SUBTASKS MAPREDUCE-4739. Some MapReduce tests fail to find winutils. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index fef0902c8d..80ae926c56 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -336,17 +336,6 @@ public static Path getStartJobCommitFile(Configuration conf, String user, return startCommitFile; } - private static long[] parseTimeStamps(String[] strs) { - if (null == strs) { - return null; - } - long[] result = new long[strs.length]; - for(int i=0; i < strs.length; ++i) { - result[i] = Long.parseLong(strs[i]); - } - return result; - } - public static void setupDistributedCache( Configuration conf, Map localResources) @@ -356,7 +345,7 @@ public static void setupDistributedCache( parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.ARCHIVE, DistributedCache.getCacheArchives(conf), - parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)), + DistributedCache.getArchiveTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), DistributedCache.getArchiveVisibilities(conf)); @@ -365,7 +354,7 @@ public static void setupDistributedCache( localResources, LocalResourceType.FILE, DistributedCache.getCacheFiles(conf), - parseTimeStamps(DistributedCache.getFileTimestamps(conf)), + DistributedCache.getFileTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES), DistributedCache.getFileVisibilities(conf)); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java index 065088d4fa..ce40bd1fc1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java @@ -245,14 +245,14 @@ public void testDeprecatedFunctions() throws Exception { Assert.assertEquals(1, DistributedCache.getArchiveTimestamps(conf).length); Assert.assertEquals(1234567890, - Long.parseLong(DistributedCache.getArchiveTimestamps(conf)[0])); + DistributedCache.getArchiveTimestamps(conf)[0]); DistributedCache.setFileTimestamps(conf, "1234567890"); Assert.assertEquals(1234567890, conf.getLong(DistributedCache.CACHE_FILES_TIMESTAMPS, 0)); Assert.assertEquals(1, DistributedCache.getFileTimestamps(conf).length); Assert.assertEquals(1234567890, - Long.parseLong(DistributedCache.getFileTimestamps(conf)[0])); + DistributedCache.getFileTimestamps(conf)[0]); DistributedCache.createAllSymlink(conf, new File("Test Job Cache Dir"), new File("Test Work Dir")); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java index 903ad085c7..eaa5af8672 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java @@ -214,17 +214,34 @@ public static Path[] getLocalCacheFiles(Configuration conf) return StringUtils.stringToPath(conf.getStrings(MRJobConfig.CACHE_LOCALFILES)); } + /** + * Parse a list of strings into longs. + * @param strs the list of strings to parse + * @return a list of longs that were parsed. same length as strs. + */ + private static long[] parseTimestamps(String[] strs) { + if (strs == null) { + return null; + } + long[] result = new long[strs.length]; + for(int i=0; i < strs.length; ++i) { + result[i] = Long.parseLong(strs[i]); + } + return result; + } + /** * Get the timestamps of the archives. Used by internal * DistributedCache and MapReduce code. * @param conf The configuration which stored the timestamps - * @return a string array of timestamps + * @return a long array of timestamps * @throws IOException * @deprecated Use {@link JobContext#getArchiveTimestamps()} instead */ @Deprecated - public static String[] getArchiveTimestamps(Configuration conf) { - return conf.getStrings(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS); + public static long[] getArchiveTimestamps(Configuration conf) { + return parseTimestamps( + conf.getStrings(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS)); } @@ -232,13 +249,14 @@ public static String[] getArchiveTimestamps(Configuration conf) { * Get the timestamps of the files. Used by internal * DistributedCache and MapReduce code. * @param conf The configuration which stored the timestamps - * @return a string array of timestamps + * @return a long array of timestamps * @throws IOException * @deprecated Use {@link JobContext#getFileTimestamps()} instead */ @Deprecated - public static String[] getFileTimestamps(Configuration conf) { - return conf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS); + public static long[] getFileTimestamps(Configuration conf) { + return parseTimestamps( + conf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS)); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java index fb48ad900a..b4c6dca554 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java @@ -342,7 +342,23 @@ public Path[] getLocalCacheFiles() public Path[] getFileClassPaths() { return DistributedCache.getFileClassPaths(conf); } - + + /** + * Parse a list of longs into strings. + * @param timestamps the list of longs to parse + * @return a list of string that were parsed. same length as timestamps. + */ + private static String[] toTimestampStrs(long[] timestamps) { + if (timestamps == null) { + return null; + } + String[] result = new String[timestamps.length]; + for(int i=0; i < timestamps.length; ++i) { + result[i] = Long.toString(timestamps[i]); + } + return result; + } + /** * Get the timestamps of the archives. Used by internal * DistributedCache and MapReduce code. @@ -350,7 +366,7 @@ public Path[] getFileClassPaths() { * @throws IOException */ public String[] getArchiveTimestamps() { - return DistributedCache.getArchiveTimestamps(conf); + return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf)); } /** @@ -360,7 +376,7 @@ public String[] getArchiveTimestamps() { * @throws IOException */ public String[] getFileTimestamps() { - return DistributedCache.getFileTimestamps(conf); + return toTimestampStrs(DistributedCache.getFileTimestamps(conf)); } /**