YARN-8223. Improved yarn auxiliary service to load jar file from HDFS.
Contributed by Zian Chen
This commit is contained in:
parent
6795f8072f
commit
8cdb032aff
@ -67,6 +67,50 @@ The collector class configuration may specify a comma-separated list of collecto
|
|||||||
|:---- |:---- |:---- |
|
|:---- |:---- |:---- |
|
||||||
| `yarn.nodemanager.aux-services` | `...,mapreduce_shuffle` | The auxiliary service name |
|
| `yarn.nodemanager.aux-services` | `...,mapreduce_shuffle` | The auxiliary service name |
|
||||||
| `yarn.nodemanager.aux-services.mapreduce_shuffle.class` | `org.apache.hadoop.mapred.ShuffleHandler` | The auxiliary service class to use |
|
| `yarn.nodemanager.aux-services.mapreduce_shuffle.class` | `org.apache.hadoop.mapred.ShuffleHandler` | The auxiliary service class to use |
|
||||||
|
| `yarn.nodemanager.aux-services.%s.classpath` | NONE | local directory which includes the related jar file as well as all the dependencies’ jar file. We could specify the single jar file or use /dep/* to load all jars under the dep directory. |
|
||||||
|
| `yarn.nodemanager.aux-services.%s.remote-classpath` | NONE | The remote absolute or relative path to jar file |
|
||||||
|
|
||||||
|
#### Example of loading jar file from HDFS:
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>yarn.nodemanager.aux-services</name>
|
||||||
|
<value>mapreduce_shuffle,AuxServiceFromHDFS</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.remote-classpath</name>
|
||||||
|
<value>/aux/test/aux-service-hdfs.jar</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.class</name>
|
||||||
|
<value>org.apache.auxtest.AuxServiceFromHDFS2</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Example of loading jar file from local file system:
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>yarn.nodemanager.aux-services</name>
|
||||||
|
<value>mapreduce_shuffle,AuxServiceFromHDFS</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.classpath</name>
|
||||||
|
<value>/aux/test/aux-service-hdfs.jar</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>yarn.nodemanager.aux-services.AuxServiceFromHDFS.class</name>
|
||||||
|
<value>org.apache.auxtest.AuxServiceFromHDFS2</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
|
```
|
||||||
|
|
||||||
**IMPORTANT:** If setting an auxiliary service in addition the default
|
**IMPORTANT:** If setting an auxiliary service in addition the default
|
||||||
`mapreduce_shuffle` service, then a new service key should be added to the
|
`mapreduce_shuffle` service, then a new service key should be added to the
|
||||||
|
@ -230,15 +230,30 @@ public void serviceInit(Configuration conf) throws Exception {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (reDownload) {
|
if (reDownload) {
|
||||||
|
LocalResourceType srcType = null;
|
||||||
|
String lowerDst = StringUtils.toLowerCase(src.toString());
|
||||||
|
if (lowerDst.endsWith(".jar")) {
|
||||||
|
srcType = LocalResourceType.FILE;
|
||||||
|
} else if (lowerDst.endsWith(".zip") ||
|
||||||
|
lowerDst.endsWith(".tar.gz") || lowerDst.endsWith(".tgz")
|
||||||
|
|| lowerDst.endsWith(".tar")) {
|
||||||
|
srcType = LocalResourceType.ARCHIVE;
|
||||||
|
} else {
|
||||||
|
throw new YarnRuntimeException(
|
||||||
|
"Can not unpack file from remote-file-path:" + src
|
||||||
|
+ "for aux-service:" + ".\n");
|
||||||
|
}
|
||||||
LocalResource scRsrc = LocalResource.newInstance(
|
LocalResource scRsrc = LocalResource.newInstance(
|
||||||
URL.fromURI(src.toUri()),
|
URL.fromURI(src.toUri()),
|
||||||
LocalResourceType.ARCHIVE, LocalResourceVisibility.PRIVATE,
|
srcType, LocalResourceVisibility.PRIVATE,
|
||||||
scFileStatus.getLen(), scFileStatus.getModificationTime());
|
scFileStatus.getLen(), scFileStatus.getModificationTime());
|
||||||
FSDownload download = new FSDownload(localLFS, null, conf,
|
FSDownload download = new FSDownload(localLFS, null, conf,
|
||||||
downloadDest, scRsrc, null);
|
downloadDest, scRsrc, null);
|
||||||
try {
|
try {
|
||||||
Path downloaded = download.call();
|
Path downloaded = download.call();
|
||||||
dest = new Path(downloaded + Path.SEPARATOR + "*");
|
// don't need to convert downloaded path into a dir
|
||||||
|
// since its already a jar path.
|
||||||
|
dest = downloaded;
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new YarnRuntimeException(
|
throw new YarnRuntimeException(
|
||||||
"Exception happend while downloading files "
|
"Exception happend while downloading files "
|
||||||
|
Loading…
Reference in New Issue
Block a user