From 241398de3b2c21beae7008f1c1a35761a4c06531 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:06:07 +0800 Subject: [PATCH] YARN-11492. Improve createJerseyClient#setConnectTimeout Code. (#5636). Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Ayush Saxena --- .../router/webapp/RouterWebServiceUtil.java | 34 ++++++-- .../webapp/TestRouterWebServiceUtil.java | 84 +++++++++++++++++++ 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index 07afc9180a..724fa329f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -363,14 +363,32 @@ public static AppsInfo mergeAppsInfo(ArrayList appsInfo, */ protected static Client createJerseyClient(Configuration conf) { Client client = Client.create(); - client.setConnectTimeout((int) conf - .getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, - YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT, - TimeUnit.MILLISECONDS)); - client.setReadTimeout((int) conf - .getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, - YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT, - TimeUnit.MILLISECONDS)); + + long checkConnectTimeOut = conf.getLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, 0); + int connectTimeOut = (int) conf.getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS); + if (checkConnectTimeOut <= 0 || checkConnectTimeOut > Integer.MAX_VALUE) { + LOG.warn("Configuration {} = {} ms error. We will use the default value({} ms).", + YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, connectTimeOut, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT); + connectTimeOut = (int) TimeUnit.MILLISECONDS.convert( + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS); + } + client.setConnectTimeout(connectTimeOut); + + long checkReadTimeout = conf.getLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, 0); + int readTimeout = (int) conf.getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT, TimeUnit.MILLISECONDS); + + if (checkReadTimeout < 0) { + LOG.warn("Configuration {} = {} ms error. We will use the default value({} ms).", + YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, connectTimeOut, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT); + readTimeout = (int) TimeUnit.MILLISECONDS.convert( + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS); + } + client.setReadTimeout(readTimeout); + return client; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java index 96a6881adc..9ce44250eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java @@ -20,10 +20,15 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Random; +import java.util.concurrent.TimeUnit; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.config.ClientConfig; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; @@ -678,4 +683,83 @@ public void testMergeDiffApplicationStatisticsInfo() { Assert.assertEquals(YarnApplicationState.FINISHED, item3Result.getState()); Assert.assertEquals(item4.getCount(), item3Result.getCount()); } + + @Test + public void testCreateJerseyClient() { + // Case1, default timeout, The default timeout is 30s. + YarnConfiguration configuration = new YarnConfiguration(); + Client client01 = RouterWebServiceUtil.createJerseyClient(configuration); + Map properties = client01.getProperties(); + int readTimeOut = (int) properties.get(ClientConfig.PROPERTY_READ_TIMEOUT); + int connectTimeOut = (int) properties.get(ClientConfig.PROPERTY_CONNECT_TIMEOUT); + Assert.assertEquals(30000, readTimeOut); + Assert.assertEquals(30000, connectTimeOut); + client01.destroy(); + + // Case2, set a negative timeout, We'll get the default timeout(30s) + YarnConfiguration configuration2 = new YarnConfiguration(); + configuration2.setLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, -1L); + configuration2.setLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, -1L); + Client client02 = RouterWebServiceUtil.createJerseyClient(configuration2); + Map properties02 = client02.getProperties(); + int readTimeOut02 = (int) properties02.get(ClientConfig.PROPERTY_READ_TIMEOUT); + int connectTimeOut02 = (int) properties02.get(ClientConfig.PROPERTY_CONNECT_TIMEOUT); + Assert.assertEquals(30000, readTimeOut02); + Assert.assertEquals(30000, connectTimeOut02); + client02.destroy(); + + // Case3, Set the maximum value that exceeds the integer + // We'll get the default timeout(30s) + YarnConfiguration configuration3 = new YarnConfiguration(); + long connectTimeOutLong = (long) Integer.MAX_VALUE + 1; + long readTimeOutLong = (long) Integer.MAX_VALUE + 1; + + configuration3.setLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, connectTimeOutLong); + configuration3.setLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, readTimeOutLong); + Client client03 = RouterWebServiceUtil.createJerseyClient(configuration3); + Map properties03 = client03.getProperties(); + int readTimeOut03 = (int) properties03.get(ClientConfig.PROPERTY_READ_TIMEOUT); + int connectTimeOut03 = (int) properties03.get(ClientConfig.PROPERTY_CONNECT_TIMEOUT); + Assert.assertEquals(30000, readTimeOut03); + Assert.assertEquals(30000, connectTimeOut03); + client03.destroy(); + } + + @Test + public void testJerseyClient() { + // Case1, Set to negative 1. + YarnConfiguration conf = new YarnConfiguration(); + conf.setLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, -1L); + conf.setLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, -1L); + + int connectTimeOut = (int) getTimeDuration(conf, + YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT); + int readTimeout = (int) getTimeDuration(conf, + YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT); + Assert.assertEquals(-1, connectTimeOut); + Assert.assertEquals(-1, readTimeout); + + // Case2, Set the maximum value that exceeds the integer. + // Converted to int, there will be a value out of bounds. + YarnConfiguration conf1 = new YarnConfiguration(); + long connectTimeOutLong = (long) Integer.MAX_VALUE + 1; + long readTimeOutLong = (long) Integer.MAX_VALUE + 1; + conf1.setLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, connectTimeOutLong); + conf1.setLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, readTimeOutLong); + + int connectTimeOut1 = (int) getTimeDuration(conf1, + YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT); + int readTimeout1 = (int) getTimeDuration(conf1, + YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT); + Assert.assertEquals(-2147483648, connectTimeOut1); + Assert.assertEquals(-2147483648, readTimeout1); + } + + private long getTimeDuration(YarnConfiguration conf, String varName, long defaultValue) { + return conf.getTimeDuration(varName, defaultValue, TimeUnit.MILLISECONDS); + } } \ No newline at end of file