From f91e21ac109e753e76d19c5c872c59a767b7b837 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Mon, 27 Apr 2020 08:26:11 +0200 Subject: [PATCH] YARN-10194. YARN RMWebServices /scheduler-conf/validate leaks ZK Connections. Contributed by Prabhu Joseph --- .../CapacitySchedulerConfigValidator.java | 18 +++++++++++------- .../conf/FSSchedulerConfigurationStore.java | 1 + .../conf/InMemoryConfigurationStore.java | 6 ++++++ .../capacity/conf/YarnConfigurationStore.java | 2 +- .../capacity/conf/ZKConfigurationStore.java | 8 ++++++++ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index 676c6d177b..1c598efd8d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -42,14 +42,18 @@ private CapacitySchedulerConfigValidator() { public static boolean validateCSConfiguration( final Configuration oldConf, final Configuration newConf, final RMContext rmContext) throws IOException { - //TODO: extract all the validation steps and replace reinitialize with - //the specific validation steps CapacityScheduler newCs = new CapacityScheduler(); - newCs.setConf(oldConf); - newCs.setRMContext(rmContext); - newCs.init(oldConf); - newCs.reinitialize(newConf, rmContext, true); - return true; + try { + //TODO: extract all the validation steps and replace reinitialize with + //the specific validation steps + newCs.setConf(oldConf); + newCs.setRMContext(rmContext); + newCs.init(oldConf); + newCs.reinitialize(newConf, rmContext, true); + return true; + } finally { + newCs.stop(); + } } public static Set validatePlacementRules( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java index 97d9933109..3aee7c648c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java @@ -358,6 +358,7 @@ protected Version getCurrentVersion() { return CURRENT_VERSION_INFO; } + @Override public void close() throws IOException { if (fileSystem != null) { fileSystem.close(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java index 45d19564ea..0fc0ecc41f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import java.io.IOException; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -150,4 +151,9 @@ public Version getCurrentVersion() { public void checkVersion() { // Does nothing. (Version is always compatible since it's in memory) } + + @Override + public void close() throws IOException { + // Does nothing. + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 8ab3e4437b..4480bc34dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -99,7 +99,7 @@ public abstract void initialize(Configuration conf, Configuration schedConf, * Closes the configuration store, releasing any required resources. * @throws IOException on failure to close */ - public void close() throws IOException {} + public abstract void close() throws IOException; /** * Logs the configuration change to backing store. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index 8c7ed375ab..cd7a166303 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -31,6 +31,7 @@ import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.data.ACL; +import java.io.IOException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; @@ -316,4 +317,11 @@ private static Object deserializeObject(byte[] bytes) throws Exception { private static T unsafeCast(Object o) throws ClassCastException { return (T)o; } + + @Override + public void close() throws IOException { + if (zkManager != null) { + zkManager.close(); + } + } }