big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9147. Rmove auxiliary services when manifest file is removed.

Browse Source 
Contributed by Billie Rinaldi
This commit is contained in:
Eric Yang 2019-01-03 12:54:16 -05:00
parent ecdeaa7e6a
commit dfceffa70d
2 changed files with 35 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
View File

@ -40,6 +40,7 @@
import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceConfiguration;
@ -413,6 +414,7 @@ private synchronized void maybeRemoveAuxService(String sName) {
serviceRecordMap.remove(sName); serviceRecordMap.remove(sName);
serviceMetaData.remove(sName); serviceMetaData.remove(sName);
if (s != null) { if (s != null) {
LOG.info("Removing aux service " + sName);
stopAuxService(s); stopAuxService(s);
} }
} }
@ -557,8 +559,24 @@ private synchronized AuxServiceRecords maybeReadManifestFile() throws
* @param startServices if true starts services, otherwise only inits services * @param startServices if true starts services, otherwise only inits services
* @throws IOException * @throws IOException
*/ */
private synchronized void loadManifest(Configuration conf, boolean @VisibleForTesting
protected synchronized void loadManifest(Configuration conf, boolean
startServices) throws IOException { startServices) throws IOException {
if (manifest == null) {
return;
}
if (!manifestFS.exists(manifest)) {
if (serviceMap.isEmpty()) {
return;
}
LOG.info("Manifest file " + manifest + " doesn't exist, stopping " +
"auxiliary services");
Set<String> servicesToRemove = new HashSet<>(serviceMap.keySet());
for (String sName : servicesToRemove) {
maybeRemoveAuxService(sName);
}
return;
}
AuxServiceRecords services = maybeReadManifestFile(); AuxServiceRecords services = maybeReadManifestFile();
if (services == null) { if (services == null) {
// read did not occur or no changes detected // read did not occur or no changes detected
@ -596,15 +614,10 @@ private synchronized void loadManifest(Configuration conf, boolean
} }
// remove aux services that do not appear in the manifest // remove aux services that do not appear in the manifest
List<String> servicesToRemove = new ArrayList<>(); Set<String> servicesToRemove = new HashSet<>(serviceMap.keySet());
for (String sName : serviceMap.keySet()) { servicesToRemove.removeAll(loadedAuxServices);
if (!loadedAuxServices.contains(sName)) {
foundChanges = true;
servicesToRemove.add(sName);
}
}
for (String sName : servicesToRemove) { for (String sName : servicesToRemove) {
LOG.info("Removing aux service " + sName); foundChanges = true;
maybeRemoveAuxService(sName); maybeRemoveAuxService(sName);
} }

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
View File

@ -885,4 +885,17 @@ public void testAuxServicesManifestPermissions() throws IOException {
aux.init(conf); aux.init(conf);
assertEquals(2, aux.getServices().size()); assertEquals(2, aux.getServices().size());
} }
@Test
public void testRemoveManifest() throws IOException {
Assume.assumeTrue(useManifest);
Configuration conf = getABConf();
final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER,
MOCK_CONTEXT, MOCK_DEL_SERVICE);
aux.init(conf);
assertEquals(2, aux.getServices().size());
manifest.delete();
aux.loadManifest(conf, false);
assertEquals(0, aux.getServices().size());
}
} }