diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md index 6b1caf4182..049286748b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md @@ -64,7 +64,10 @@ The collector class configuration may specify a comma-separated list of collecto ### NodeManager Configuration properties, `yarn-site.xml` in all nodes: There are two ways to configure auxiliary services, through a manifest file or through the Configuration (the old way). If a manifest file is used, auxiliary service configurations are not read from the Configuration. -If using a manifest file, the file name should be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms`. Otherwise, set the following properties to configure aux services through the Configuration. + +If using a manifest file, the file name can be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`, or the file may be sent to the NM via a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`. If the file name is set in the Configuration, NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 2 minutes; setting interval <= 0 means it will not be reloaded automatically). + +Otherwise, set the following properties to configure aux services through the Configuration. | **Property** | **Default Value** | **Explanation** | |:---- |:---- |:---- | diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index cde0683814..01d901a4cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -484,6 +484,22 @@ private void reloadManifest() throws IOException { loadManifest(getConfig(), true); } + /** + * Reloads auxiliary services. Must be called after service init. + * + * @param services a list of auxiliary services + * @throws IOException if aux services have not been started yet + */ + public void reload(AuxServiceRecords services) throws IOException { + if (getServiceState() != Service.STATE.STARTED) { + throw new IOException("Auxiliary services have not been started yet, " + + "please retry later"); + } + LOG.info("Received list of auxiliary services: " + mapper + .writeValueAsString(services)); + loadServices(services, getConfig(), true); + } + private boolean checkManifestPermissions(FileStatus status) throws IOException { if ((status.getPermission().toShort() & 0022) != 0) { @@ -578,6 +594,19 @@ protected synchronized void loadManifest(Configuration conf, boolean return; } AuxServiceRecords services = maybeReadManifestFile(); + loadServices(services, conf, startServices); + } + + /** + * Updates current aux services based on changes found in the service list. + * + * @param services list of auxiliary services + * @param conf configuration + * @param startServices if true starts services, otherwise only inits services + * @throws IOException + */ + private synchronized void loadServices(AuxServiceRecords services, + Configuration conf, boolean startServices) throws IOException { if (services == null) { // read did not occur or no changes detected return; @@ -613,7 +642,7 @@ protected synchronized void loadManifest(Configuration conf, boolean } } - // remove aux services that do not appear in the manifest + // remove aux services that do not appear in the new list Set servicesToRemove = new HashSet<>(serviceMap.keySet()); servicesToRemove.removeAll(loadedAuxServices); for (String sName : servicesToRemove) { @@ -622,7 +651,7 @@ protected synchronized void loadManifest(Configuration conf, boolean } if (!foundChanges) { - LOG.info("No auxiliary services changes detected in manifest"); + LOG.info("No auxiliary services changes detected"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index b825375aa2..2267e072a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -31,6 +31,7 @@ import java.util.Set; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AuxiliaryServicesInfo; @@ -575,6 +576,27 @@ public AuxiliaryServicesInfo getAuxiliaryServices(@javax.ws.rs.core.Context return auxiliaryServices; } + @PUT + @Path("/auxiliaryservices") + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public Response putAuxiliaryServices(@javax.ws.rs.core.Context + HttpServletRequest req, AuxServiceRecords services) { + if (!hasAdminAccess(req)) { + return Response.status(Status.FORBIDDEN).build(); + } + if (services == null) { + return Response.status(Status.BAD_REQUEST).build(); + } + try { + nmContext.getAuxServices().reload(services); + } catch (Exception e) { + LOG.error("Fail to reload auxiliary services, reason: ", e); + return Response.status(Status.INTERNAL_SERVER_ERROR).entity(e).build(); + } + return Response.ok().build(); + } + @PUT @Path("/yarn/sysfs/{user}/{appId}") @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index cb8abc79ce..6cf2b7e6a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -898,4 +898,27 @@ public void testRemoveManifest() throws IOException { aux.loadManifest(conf, false); assertEquals(0, aux.getServices().size()); } + + @Test + public void testManualReload() throws IOException { + Configuration conf = getABConf(); + final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, + MOCK_CONTEXT, MOCK_DEL_SERVICE); + aux.init(conf); + try { + aux.reload(null); + Assert.fail("Should receive the exception."); + } catch (IOException e) { + assertTrue("Wrong message: " + e.getMessage(), + e.getMessage().equals("Auxiliary services have not been started " + + "yet, please retry later")); + } + aux.start(); + assertEquals(2, aux.getServices().size()); + aux.reload(null); + assertEquals(2, aux.getServices().size()); + aux.reload(new AuxServiceRecords()); + assertEquals(0, aux.getServices().size()); + aux.stop(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md index d21119854e..b31627dbbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md @@ -106,7 +106,8 @@ To launch auxiliary services on a NodeManager, users have to add their jar to No ### Manifest This section describes the auxiliary service manifest for aux-service classpath isolation. -The manifest file should be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms`. +The manifest file can be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 2 minutes; setting interval <= 0 means it will not be reloaded automatically). +Alternatively, the manifest file may be sent to the NM via REST API by making a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`. An example manifest that configures classpath isolation for a CustomAuxService follows. One or more files may be specified to make up the classpath of a service, with jar or archive formats being supported. ``` diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md index 03bf4f228c..1d03223d1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md @@ -591,13 +591,16 @@ Auxiliary Services API With the auxiliary services API, you can obtain a collection of resources, each of which represents an auxiliary service. When you run a GET operation on this resource, you obtain a collection of auxiliary service information objects. +A YARN admin can use a PUT operation to update the auxiliary services running on the NodeManager. The body of the request should be of the same format as an auxiliary services manifest file. + ### URI * http://nm-http-address:port/ws/v1/node/auxiliaryservices ### HTTP Operations Supported - * GET + * GET + * PUT ### Query Parameters Supported @@ -611,7 +614,7 @@ When you make a request for the list of auxiliary services, the information will |:---- |:---- |:---- | | services | array of service information objects(JSON)/zero or more service information objects (XML) | A collection of service information objects | -### Response Examples +### GET Response Examples **JSON response**