diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8261b34d17..55284da2f9 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -736,6 +736,9 @@ Release 2.7.3 - UNRELEASED MAPREDUCE-6554. MRAppMaster servicestart failing with NPE in MRAppMaster#parsePreviousJobHistory (Bibin A Chundatt via jlowe) + MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. + (Xuan Gong via jlowe) + Release 2.7.2 - 2016-01-25 INCOMPATIBLE CHANGES @@ -1041,6 +1044,9 @@ Release 2.6.4 - UNRELEASED MAPREDUCE-6554. MRAppMaster servicestart failing with NPE in MRAppMaster#parsePreviousJobHistory (Bibin A Chundatt via jlowe) + MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. + (Xuan Gong via jlowe) + Release 2.6.3 - 2015-12-17 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java index 4335c82296..93ea5c437c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java @@ -22,11 +22,11 @@ import java.security.PrivilegedAction; import java.util.HashMap; import java.util.Map; - import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; @@ -97,4 +97,26 @@ public MRClientProtocol run() { } }); } + + public void close() throws IOException { + if (rm != null) { + rm.close(); + } + + if (hsProxy != null) { + RPC.stopProxy(hsProxy); + hsProxy = null; + } + + if (cache != null && !cache.isEmpty()) { + for (ClientServiceDelegate delegate : cache.values()) { + if (delegate != null) { + delegate.close(); + delegate = null; + } + } + cache.clear(); + cache = null; + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java index 8517c19737..eac8dbc0ae 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java @@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -532,4 +533,19 @@ public LogParams getLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID) throw new IOException("Cannot get log path for a in-progress job"); } } + + public void close() throws IOException { + if (rm != null) { + rm.close(); + } + + if (historyServerProxy != null) { + RPC.stopProxy(historyServerProxy); + } + + if (realProxy != null) { + RPC.stopProxy(realProxy); + realProxy = null; + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index fe631de99a..1342282784 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -756,4 +756,15 @@ private static void warnForJavaLibPath(String opts, String component, envConf + " config settings."); } } + + public void close() throws IOException { + if (resMgrDelegate != null) { + resMgrDelegate.close(); + resMgrDelegate = null; + } + if (clientCache != null) { + clientCache.close(); + clientCache = null; + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java index 29d3f66cc3..6583fa29ad 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java @@ -44,7 +44,8 @@ public ClientProtocol create(InetSocketAddress addr, Configuration conf) @Override public void close(ClientProtocol clientProtocol) throws IOException { - // nothing to do + if (clientProtocol instanceof YARNRunner) { + ((YARNRunner)clientProtocol).close(); + } } - } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java index aeb20cd449..4d84fa9e10 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java @@ -21,10 +21,11 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.doNothing; import java.io.IOException; import java.nio.ByteBuffer; - import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; @@ -113,6 +114,8 @@ public void testClusterGetDelegationToken() throws Exception { @Override protected void serviceStart() throws Exception { assertTrue(this.client instanceof YarnClientImpl); + this.client = spy(this.client); + doNothing().when(this.client).close(); ((YarnClientImpl) this.client).setRMClient(cRMProtocol); } }; @@ -126,5 +129,4 @@ protected void serviceStart() throws Exception { } } } - }