diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java index 48b157ee26..48cc29e702 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java @@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.InvalidJobConfException; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -144,10 +143,9 @@ void createLocalResources(Map localResources) LocalResource orig = localResources.get(linkName); if(orig != null && !orig.getResource().equals(URL.fromURI(p.toUri()))) { - throw new InvalidJobConfException( - getResourceDescription(orig.getType()) + orig.getResource() - + - " conflicts with " + getResourceDescription(type) + u); + LOG.warn(getResourceDescription(orig.getType()) + orig.getResource() + + " conflicts with " + getResourceDescription(type) + u); + continue; } Boolean sharedCachePolicy = sharedCacheUploadPolicies.get(u.toString()); sharedCachePolicy = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 3aadd637bf..c6a287439d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -360,7 +360,7 @@ public void testSetupDistributedCacheEmpty() throws IOException { } @SuppressWarnings("deprecation") - @Test(timeout = 120000, expected = InvalidJobConfException.class) + @Test(timeout = 120000) public void testSetupDistributedCacheConflicts() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); @@ -388,10 +388,18 @@ public void testSetupDistributedCacheConflicts() throws Exception { Map localResources = new HashMap(); MRApps.setupDistributedCache(conf, localResources); + + assertEquals(1, localResources.size()); + LocalResource lr = localResources.get("something"); + //Archive wins + assertNotNull(lr); + assertEquals(10l, lr.getSize()); + assertEquals(10l, lr.getTimestamp()); + assertEquals(LocalResourceType.ARCHIVE, lr.getType()); } @SuppressWarnings("deprecation") - @Test(timeout = 120000, expected = InvalidJobConfException.class) + @Test(timeout = 120000) public void testSetupDistributedCacheConflictsFiles() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); @@ -416,6 +424,14 @@ public void testSetupDistributedCacheConflictsFiles() throws Exception { Map localResources = new HashMap(); MRApps.setupDistributedCache(conf, localResources); + + assertEquals(1, localResources.size()); + LocalResource lr = localResources.get("something"); + //First one wins + assertNotNull(lr); + assertEquals(10l, lr.getSize()); + assertEquals(10l, lr.getTimestamp()); + assertEquals(LocalResourceType.FILE, lr.getType()); } @SuppressWarnings("deprecation")