From b3b4d4ccb53fdf8dacc66e912822b34f8b3bf215 Mon Sep 17 00:00:00 2001
From: Wangda Tan <wangda@apache.org>
Date: Thu, 19 Jul 2018 12:03:24 -0700
Subject: [PATCH] MAPREDUCE-7118. Distributed cache conflicts breaks backwards
 compatability. (Jason Lowe via wangda)

Change-Id: I89ab4852b4ad305fec19812e8931c59d96581376
---
 .../v2/util/LocalResourceBuilder.java         |  8 +++-----
 .../hadoop/mapreduce/v2/util/TestMRApps.java  | 20 +++++++++++++++++--
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java
index 48b157ee26..48cc29e702 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/LocalResourceBuilder.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.InvalidJobConfException;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -144,10 +143,9 @@ void createLocalResources(Map<String, LocalResource> localResources)
 
         LocalResource orig = localResources.get(linkName);
         if(orig != null && !orig.getResource().equals(URL.fromURI(p.toUri()))) {
-          throw new InvalidJobConfException(
-              getResourceDescription(orig.getType()) + orig.getResource()
-                  +
-              " conflicts with " + getResourceDescription(type) + u);
+          LOG.warn(getResourceDescription(orig.getType()) + orig.getResource()
+              + " conflicts with " + getResourceDescription(type) + u);
+          continue;
         }
         Boolean sharedCachePolicy = sharedCacheUploadPolicies.get(u.toString());
         sharedCachePolicy =
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
index 3aadd637bf..c6a287439d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
@@ -360,7 +360,7 @@ public void testSetupDistributedCacheEmpty() throws IOException {
   }
   
   @SuppressWarnings("deprecation")
-  @Test(timeout = 120000, expected = InvalidJobConfException.class)
+  @Test(timeout = 120000)
   public void testSetupDistributedCacheConflicts() throws Exception {
     Configuration conf = new Configuration();
     conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
@@ -388,10 +388,18 @@ public void testSetupDistributedCacheConflicts() throws Exception {
     Map<String, LocalResource> localResources = 
       new HashMap<String, LocalResource>();
     MRApps.setupDistributedCache(conf, localResources);
+
+    assertEquals(1, localResources.size());
+    LocalResource lr = localResources.get("something");
+    //Archive wins
+    assertNotNull(lr);
+    assertEquals(10l, lr.getSize());
+    assertEquals(10l, lr.getTimestamp());
+    assertEquals(LocalResourceType.ARCHIVE, lr.getType());
   }
   
   @SuppressWarnings("deprecation")
-  @Test(timeout = 120000, expected = InvalidJobConfException.class)
+  @Test(timeout = 120000)
   public void testSetupDistributedCacheConflictsFiles() throws Exception {
     Configuration conf = new Configuration();
     conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
@@ -416,6 +424,14 @@ public void testSetupDistributedCacheConflictsFiles() throws Exception {
     Map<String, LocalResource> localResources = 
       new HashMap<String, LocalResource>();
     MRApps.setupDistributedCache(conf, localResources);
+
+    assertEquals(1, localResources.size());
+    LocalResource lr = localResources.get("something");
+    //First one wins
+    assertNotNull(lr);
+    assertEquals(10l, lr.getSize());
+    assertEquals(10l, lr.getTimestamp());
+    assertEquals(LocalResourceType.FILE, lr.getType());
   }
   
   @SuppressWarnings("deprecation")