diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md index 049286748b..441bf6a846 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md @@ -65,7 +65,7 @@ The collector class configuration may specify a comma-separated list of collecto There are two ways to configure auxiliary services, through a manifest file or through the Configuration (the old way). If a manifest file is used, auxiliary service configurations are not read from the Configuration. -If using a manifest file, the file name can be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`, or the file may be sent to the NM via a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`. If the file name is set in the Configuration, NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 2 minutes; setting interval <= 0 means it will not be reloaded automatically). +If using a manifest, the feature must be enabled by setting the property `yarn.nodemanager.aux-services.manifest.enabled` to true in *yarn-site.xml*. The file path can be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`, or the file may be sent to each NM via a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`. If the file path is set in the Configuration, NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 0; setting interval <= 0 means it will not be reloaded automatically). Otherwise, set the following properties to configure aux services through the Configuration. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index c8ac82a6ea..2be73e1d96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2227,13 +2227,34 @@ public static boolean isAclEnabled(Configuration conf) { public static final String NM_AUX_SERVICES = NM_PREFIX + "aux-services"; + /** + * Boolean indicating whether loading aux services from a manifest is + * enabled. If enabled, aux services may be dynamically modified through + * reloading the manifest via filesystem changes or a REST API. When + * enabled, aux services configuration properties unrelated to the manifest + * will be ignored. + */ + public static final String NM_AUX_SERVICES_MANIFEST_ENABLED = + NM_AUX_SERVICES + ".manifest.enabled"; + + public static final boolean DEFAULT_NM_AUX_SERVICES_MANIFEST_ENABLED = + false; + + /** + * File containing auxiliary service specifications. + */ public static final String NM_AUX_SERVICES_MANIFEST = NM_AUX_SERVICES + ".manifest"; + /** + * Interval at which manifest file will be reloaded when modifications are + * found (0 or less means that the file will not be checked for modifications + * and reloaded). + */ public static final String NM_AUX_SERVICES_MANIFEST_RELOAD_MS = NM_AUX_SERVICES + ".manifest.reload-ms"; - public static final long DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS = 120000; + public static final long DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS = 0; public static final String NM_AUX_SERVICE_FMT = NM_PREFIX + "aux-services.%s.class"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index fea635b42b..2508c48d1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1922,9 +1922,17 @@ - A file containing auxiliary service specifications. If - manifest file is specified, yarn.nodemanager.aux-services and other - aux services configuration properties will be ignored. + Boolean indicating whether loading aux services from a manifest + is enabled. If enabled, aux services may be dynamically modified through + reloading the manifest via filesystem changes or a REST API. When + enabled, aux services configuration properties unrelated to the manifest + will be ignored. + yarn.nodemanager.aux-services.manifest.enabled + false + + + + A file containing auxiliary service specifications. yarn.nodemanager.aux-services.manifest @@ -1933,7 +1941,7 @@ Length of time in ms to wait between reloading aux services manifest. If 0 or less, manifest will not be reloaded. yarn.nodemanager.aux-services.manifest.reload-ms - + 0 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index 01d901a4cd..01611e96d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -110,8 +110,9 @@ public class AuxServices extends AbstractService private Path stateStoreRoot = null; private FileSystem stateStoreFs = null; - private Path manifest; - private FileSystem manifestFS; + private volatile boolean manifestEnabled = false; + private volatile Path manifest; + private volatile FileSystem manifestFS; private Timer manifestReloadTimer; private TimerTask manifestReloadTask; private long manifestReloadInterval; @@ -139,6 +140,13 @@ public class AuxServices extends AbstractService // Obtain services from configuration in init() } + /** + * Returns whether aux services manifest / dynamic loading is enabled. + */ + public boolean isManifestEnabled() { + return manifestEnabled; + } + /** * Adds a service to the service map. * @@ -480,7 +488,8 @@ private AuxiliaryService initAuxService(AuxServiceRecord service, * * @throws IOException if manifest can't be loaded */ - private void reloadManifest() throws IOException { + @VisibleForTesting + protected void reloadManifest() throws IOException { loadManifest(getConfig(), true); } @@ -488,9 +497,15 @@ private void reloadManifest() throws IOException { * Reloads auxiliary services. Must be called after service init. * * @param services a list of auxiliary services - * @throws IOException if aux services have not been started yet + * @throws IOException if aux services have not been started yet or dynamic + * reloading is not enabled */ - public void reload(AuxServiceRecords services) throws IOException { + public synchronized void reload(AuxServiceRecords services) throws + IOException { + if (!manifestEnabled) { + throw new IOException("Dynamic reloading is not enabled via " + + YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED); + } if (getServiceState() != Service.STATE.STARTED) { throw new IOException("Auxiliary services have not been started yet, " + "please retry later"); @@ -578,6 +593,10 @@ private synchronized AuxServiceRecords maybeReadManifestFile() throws @VisibleForTesting protected synchronized void loadManifest(Configuration conf, boolean startServices) throws IOException { + if (!manifestEnabled) { + throw new IOException("Dynamic reloading is not enabled via " + + YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED); + } if (manifest == null) { return; } @@ -730,8 +749,10 @@ public synchronized void serviceInit(Configuration conf) throws Exception { STATE_STORE_ROOT_NAME); stateStoreFs = FileSystem.getLocal(conf); } - String manifestStr = conf.get(YarnConfiguration.NM_AUX_SERVICES_MANIFEST); - if (manifestStr == null) { + manifestEnabled = conf.getBoolean( + YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED, + YarnConfiguration.DEFAULT_NM_AUX_SERVICES_MANIFEST_ENABLED); + if (!manifestEnabled) { Collection auxNames = conf.getStringCollection( YarnConfiguration.NM_AUX_SERVICES); for (final String sName : auxNames) { @@ -742,14 +763,20 @@ public synchronized void serviceInit(Configuration conf) throws Exception { addService(sName, s, service); } } else { - manifest = new Path(manifestStr); - manifestFS = FileSystem.get(new URI(manifestStr), conf); - loadManifest(conf, false); + String manifestStr = conf.get(YarnConfiguration.NM_AUX_SERVICES_MANIFEST); + if (manifestStr != null) { + manifest = new Path(manifestStr); + manifestFS = FileSystem.get(new URI(manifestStr), conf); + loadManifest(conf, false); + manifestReloadInterval = conf.getLong( + YarnConfiguration.NM_AUX_SERVICES_MANIFEST_RELOAD_MS, + YarnConfiguration.DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS); + manifestReloadTask = new ManifestReloadTask(); + } else { + LOG.info("Auxiliary services manifest is enabled, but no manifest " + + "file is specified in the configuration."); + } } - manifestReloadInterval = conf.getLong( - YarnConfiguration.NM_AUX_SERVICES_MANIFEST_RELOAD_MS, - YarnConfiguration.DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS); - manifestReloadTask = new ManifestReloadTask(); super.serviceInit(conf); } @@ -781,8 +808,10 @@ public synchronized void serviceStart() throws Exception { String name = entry.getKey(); startAuxService(name, service, serviceRecordMap.get(name)); } - if (manifest != null && manifestReloadInterval > 0) { - manifestReloadTimer = new Timer("AuxServicesManifestRelaod-Timer", + if (manifestEnabled && manifest != null && manifestReloadInterval > 0) { + LOG.info("Scheduling reloading auxiliary services manifest file at " + + "interval " + manifestReloadInterval + " ms"); + manifestReloadTimer = new Timer("AuxServicesManifestReload-Timer", true); manifestReloadTimer.schedule(manifestReloadTask, manifestReloadInterval, manifestReloadInterval); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 2267e072a6..106144f913 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -567,6 +567,10 @@ public Object getNMResourceInfo( public AuxiliaryServicesInfo getAuxiliaryServices(@javax.ws.rs.core.Context HttpServletRequest hsr) { init(); + if (!this.nmContext.getAuxServices().isManifestEnabled()) { + throw new BadRequestException("Auxiliary services manifest is not " + + "enabled"); + } AuxiliaryServicesInfo auxiliaryServices = new AuxiliaryServicesInfo(); Collection loadedServices = nmContext.getAuxServices() .getServiceRecords(); @@ -582,6 +586,11 @@ public AuxiliaryServicesInfo getAuxiliaryServices(@javax.ws.rs.core.Context MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) public Response putAuxiliaryServices(@javax.ws.rs.core.Context HttpServletRequest req, AuxServiceRecords services) { + init(); + if (!this.nmContext.getAuxServices().isManifestEnabled()) { + throw new BadRequestException("Auxiliary services manifest is not " + + "enabled"); + } if (!hasAdminAccess(req)) { return Response.status(Status.FORBIDDEN).build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index 6cf2b7e6a4..aef40911f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -252,6 +252,7 @@ public ByteBuffer getMetaData() { private void writeManifestFile(AuxServiceRecords services, Configuration conf) throws IOException { + conf.setBoolean(YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED, true); conf.set(YarnConfiguration.NM_AUX_SERVICES_MANIFEST, manifest .getAbsolutePath()); mapper.writeValue(manifest, services); @@ -901,6 +902,7 @@ public void testRemoveManifest() throws IOException { @Test public void testManualReload() throws IOException { + Assume.assumeTrue(useManifest); Configuration conf = getABConf(); final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); @@ -921,4 +923,28 @@ public void testManualReload() throws IOException { assertEquals(0, aux.getServices().size()); aux.stop(); } + + @Test + public void testReloadWhenDisabled() throws IOException { + Configuration conf = new Configuration(); + final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, + MOCK_CONTEXT, MOCK_DEL_SERVICE); + aux.init(conf); + try { + aux.reload(null); + Assert.fail("Should receive the exception."); + } catch (IOException e) { + assertTrue("Wrong message: " + e.getMessage(), + e.getMessage().equals("Dynamic reloading is not enabled via " + + YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED)); + } + try { + aux.reloadManifest(); + Assert.fail("Should receive the exception."); + } catch (IOException e) { + assertTrue("Wrong message: " + e.getMessage(), + e.getMessage().equals("Dynamic reloading is not enabled via " + + YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED)); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java index b6cc0dba4f..4ee63db817 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -33,6 +35,7 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.filter.LoggingFilter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; @@ -54,6 +57,7 @@ import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.AfterClass; import org.junit.Before; @@ -195,6 +199,7 @@ public void testNodeAuxServicesNone() throws Exception { private void addAuxServices(AuxServiceRecord... records) { AuxServices auxServices = mock(AuxServices.class); when(auxServices.getServiceRecords()).thenReturn(Arrays.asList(records)); + when(auxServices.isManifestEnabled()).thenReturn(true); nmContext.setAuxServices(auxServices); } @@ -238,7 +243,7 @@ public void testNodeHelper(String path, String media) throws Exception { } @Test - public void testNodeContainerXML() throws Exception { + public void testNodeAuxServicesXML() throws Exception { AuxServiceRecord r1 = new AuxServiceRecord().name("name1").launchTime(new Date(123L)).version("1"); AuxServiceRecord r2 = new AuxServiceRecord().name("name2").launchTime(new @@ -259,10 +264,43 @@ public void testNodeContainerXML() throws Exception { Document dom = db.parse(is); NodeList nodes = dom.getElementsByTagName("service"); assertEquals("incorrect number of elements", 2, nodes.getLength()); - verifyContainersInfoXML(nodes, r1, r2); + verifyAuxServicesInfoXML(nodes, r1, r2); } - public void verifyContainersInfoXML(NodeList nodes, AuxServiceRecord... + @Test + public void testAuxServicesDisabled() throws JSONException, Exception { + AuxServices auxServices = mock(AuxServices.class); + when(auxServices.isManifestEnabled()).thenReturn(false); + nmContext.setAuxServices(auxServices); + WebResource r = resource(); + try { + r.path("ws").path("v1").path("node").path(AUX_SERVICES_PATH) + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + fail("should have thrown exception on invalid user query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertResponseStatusCode(ClientResponse.Status.BAD_REQUEST, + response.getStatusInfo()); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch( + "exception message", + "java.lang.Exception: Auxiliary services manifest is not enabled", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + } + } + + public void verifyAuxServicesInfoXML(NodeList nodes, AuxServiceRecord... records) { for (int i = 0; i < nodes.getLength(); i++) { Element element = (Element) nodes.item(i); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md index b31627dbbf..5b9f0ef820 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md @@ -95,7 +95,7 @@ Step 5. Auxiliary services. * A simple example for the above is the auxiliary service 'ShuffleHandler' for MapReduce (MR). ShuffleHandler respects the above two requirements already, so users/admins don't have to do anything for it to support NM restart: (1) The configuration property **mapreduce.shuffle.port** controls which port the ShuffleHandler on a NodeManager host binds to, and it defaults to a non-ephemeral port. (2) The ShuffleHandler service also already supports recovery of previous state after NM restarts. - * There are two ways to configure auxiliary services, through a manifest file or through the Configuration. If a manifest file is used, auxiliary service configurations are not read from the Configuration. If no manifest file is specified, auxiliary services will be loaded via the prior method of using Configuration properties. One advantage of using the manifest file is that the NMs will detect changes in auxiliary services specified in the manifest file (as determined by the service name and version) and will remove or reload running auxiliary services as needed. To support reloading, AuxiliaryService implementations must perform any cleanup that is needed during the service stop phase for the NM to be able to create a new instance of the auxiliary service. + * There are two ways to configure auxiliary services, through a manifest or through the Configuration. Auxiliary services will only be loaded via the prior method of using Configuration properties when an auxiliary services manifest is not enabled. One advantage of using a manifest is that NMs can dynamically reload auxiliary services based on changes to the manifest. To support reloading, AuxiliaryService implementations must perform any cleanup that is needed during the service stop phase for the NM to be able to create a new instance of the auxiliary service. Auxiliary Service Classpath Isolation ------------------------------------- @@ -105,9 +105,11 @@ To launch auxiliary services on a NodeManager, users have to add their jar to No ### Manifest This section describes the auxiliary service manifest for aux-service classpath isolation. +To use a manifest, the property `yarn.nodemanager.aux-services.manifest.enabled` must be set to true in *yarn-site.xml*. -The manifest file can be set in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 2 minutes; setting interval <= 0 means it will not be reloaded automatically). -Alternatively, the manifest file may be sent to the NM via REST API by making a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`. +To load the manifest file from a filesystem, set the file path in *yarn-site.xml* under the property `yarn.nodemanager.aux-services.manifest`. The NMs will check this file for new modifications at an interval specified by `yarn.nodemanager.aux-services.manifest.reload-ms` (defaults to 0; setting interval <= 0 means it will not be reloaded automatically). +Alternatively, the manifest file may be sent to an NM via REST API by making a PUT call to the endpoint `http://nm-http-address:port/ws/v1/node/auxiliaryservices`. Note this only updates the manifest on one NM. +When it reads a new manifest, the NM will add, remove, or reload auxiliary services as needed based on the service names and versions found in the manifest. An example manifest that configures classpath isolation for a CustomAuxService follows. One or more files may be specified to make up the classpath of a service, with jar or archive formats being supported. ```