From 8cdb032aff4237d8d3970057d82290e4e32c4040 Mon Sep 17 00:00:00 2001
From: Eric Yang <eyang@apache.org>
Date: Fri, 4 May 2018 12:36:31 -0400
Subject: [PATCH] YARN-8223.  Improved yarn auxiliary service to load jar file
 from HDFS.             Contributed by Zian Chen

---
 .../PluggableShuffleAndPluggableSort.md       | 44 +++++++++++++++++++
 .../containermanager/AuxServices.java         | 19 +++++++-
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
index 5ea05673a2..9e24103f93 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
@@ -67,6 +67,50 @@ The collector class configuration may specify a comma-separated list of collecto
 |:---- |:---- |:---- |
 | `yarn.nodemanager.aux-services` | `...,mapreduce_shuffle` | The auxiliary service name |
 | `yarn.nodemanager.aux-services.mapreduce_shuffle.class` | `org.apache.hadoop.mapred.ShuffleHandler` | The auxiliary service class to use |
+| `yarn.nodemanager.aux-services.%s.classpath` | NONE | local directory which includes the related jar file as well as all the dependencies’ jar file. We could specify the single jar file or use /dep/* to load all jars under the dep directory. |
+| `yarn.nodemanager.aux-services.%s.remote-classpath` | NONE | The remote absolute or relative path to jar file |
+
+#### Example of loading jar file from HDFS:
+
+```xml
+<configuration>
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle,AuxServiceFromHDFS</value>
+    </property>
+
+    <property>
+        <name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.remote-classpath</name>
+        <value>/aux/test/aux-service-hdfs.jar</value>
+    </property>
+
+    <property>
+        <name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.class&lt;/name>
+        <value>org.apache.auxtest.AuxServiceFromHDFS2</value>
+    </property>
+</configuration>
+```
+
+#### Example of loading jar file from local file system:
+
+```xml
+<configuration>
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle,AuxServiceFromHDFS</value>
+    </property>
+
+    <property>
+        <name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.classpath</name>
+        <value>/aux/test/aux-service-hdfs.jar</value>
+    </property>
+
+    <property>
+        <name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.class&lt;/name>
+        <value>org.apache.auxtest.AuxServiceFromHDFS2</value>
+    </property>
+</configuration>
+```
 
 **IMPORTANT:** If setting an auxiliary service in addition the default
 `mapreduce_shuffle` service, then a new service key should be added to the
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index c8b7a766ba..3fe3cfd16e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -230,15 +230,30 @@ public void serviceInit(Configuration conf) throws Exception {
               }
             }
             if (reDownload) {
+              LocalResourceType srcType = null;
+              String lowerDst = StringUtils.toLowerCase(src.toString());
+              if (lowerDst.endsWith(".jar")) {
+                srcType = LocalResourceType.FILE;
+              } else if (lowerDst.endsWith(".zip") ||
+                  lowerDst.endsWith(".tar.gz") || lowerDst.endsWith(".tgz")
+                  || lowerDst.endsWith(".tar")) {
+                srcType = LocalResourceType.ARCHIVE;
+              } else {
+                throw new YarnRuntimeException(
+                    "Can not unpack file from remote-file-path:" + src
+                        + "for aux-service:" + ".\n");
+              }
               LocalResource scRsrc = LocalResource.newInstance(
                   URL.fromURI(src.toUri()),
-                  LocalResourceType.ARCHIVE, LocalResourceVisibility.PRIVATE,
+                  srcType, LocalResourceVisibility.PRIVATE,
                   scFileStatus.getLen(), scFileStatus.getModificationTime());
               FSDownload download = new FSDownload(localLFS, null, conf,
                   downloadDest, scRsrc, null);
               try {
                 Path downloaded = download.call();
-                dest = new Path(downloaded + Path.SEPARATOR + "*");
+                // don't need to convert downloaded path into a dir
+                // since its already a jar path.
+                dest = downloaded;
               } catch (Exception ex) {
                 throw new YarnRuntimeException(
                     "Exception happend while downloading files "