big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11492. Improve createJerseyClient#setConnectTimeout Code. (#5636). Contributed by Shilun Fan.

Browse Source 
Reviewed-by: Inigo Goiri <inigoiri@apache.org>
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
slfan1989 2023-06-05 19:06:07 +08:00 committed by GitHub
parent e69a077af8
commit 241398de3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 110 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
View File

@ -363,14 +363,32 @@ public static AppsInfo mergeAppsInfo(ArrayList<AppInfo> appsInfo,
*/
protected static Client createJerseyClient(Configuration conf) {
Client client = Client.create();
client.setConnectTimeout((int) conf
.getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT,
TimeUnit.MILLISECONDS));
client.setReadTimeout((int) conf
.getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT,
TimeUnit.MILLISECONDS));
long checkConnectTimeOut = conf.getLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, 0);
int connectTimeOut = (int) conf.getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS);
if (checkConnectTimeOut <= 0 || checkConnectTimeOut > Integer.MAX_VALUE) {
LOG.warn("Configuration {} = {} ms error. We will use the default value({} ms).",
YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, connectTimeOut,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT);
connectTimeOut = (int) TimeUnit.MILLISECONDS.convert(
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS);
}
client.setConnectTimeout(connectTimeOut);
long checkReadTimeout = conf.getLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, 0);
int readTimeout = (int) conf.getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT, TimeUnit.MILLISECONDS);
if (checkReadTimeout < 0) {
LOG.warn("Configuration {} = {} ms error. We will use the default value({} ms).",
YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, connectTimeOut,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT);
readTimeout = (int) TimeUnit.MILLISECONDS.convert(
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS);
}
client.setReadTimeout(readTimeout);
return client;
}

84
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
View File

@ -20,10 +20,15 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.config.ClientConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
@ -678,4 +683,83 @@ public void testMergeDiffApplicationStatisticsInfo() {
Assert.assertEquals(YarnApplicationState.FINISHED, item3Result.getState());
Assert.assertEquals(item4.getCount(), item3Result.getCount());
}
@Test
public void testCreateJerseyClient() {
// Case1, default timeout, The default timeout is 30s.
YarnConfiguration configuration = new YarnConfiguration();
Client client01 = RouterWebServiceUtil.createJerseyClient(configuration);
Map<String, Object> properties = client01.getProperties();
int readTimeOut = (int) properties.get(ClientConfig.PROPERTY_READ_TIMEOUT);
int connectTimeOut = (int) properties.get(ClientConfig.PROPERTY_CONNECT_TIMEOUT);
Assert.assertEquals(30000, readTimeOut);
Assert.assertEquals(30000, connectTimeOut);
client01.destroy();
// Case2, set a negative timeout, We'll get the default timeout(30s)
YarnConfiguration configuration2 = new YarnConfiguration();
configuration2.setLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, -1L);
configuration2.setLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, -1L);
Client client02 = RouterWebServiceUtil.createJerseyClient(configuration2);
Map<String, Object> properties02 = client02.getProperties();
int readTimeOut02 = (int) properties02.get(ClientConfig.PROPERTY_READ_TIMEOUT);
int connectTimeOut02 = (int) properties02.get(ClientConfig.PROPERTY_CONNECT_TIMEOUT);
Assert.assertEquals(30000, readTimeOut02);
Assert.assertEquals(30000, connectTimeOut02);
client02.destroy();
// Case3, Set the maximum value that exceeds the integer
// We'll get the default timeout(30s)
YarnConfiguration configuration3 = new YarnConfiguration();
long connectTimeOutLong = (long) Integer.MAX_VALUE + 1;
long readTimeOutLong = (long) Integer.MAX_VALUE + 1;
configuration3.setLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, connectTimeOutLong);
configuration3.setLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, readTimeOutLong);
Client client03 = RouterWebServiceUtil.createJerseyClient(configuration3);
Map<String, Object> properties03 = client03.getProperties();
int readTimeOut03 = (int) properties03.get(ClientConfig.PROPERTY_READ_TIMEOUT);
int connectTimeOut03 = (int) properties03.get(ClientConfig.PROPERTY_CONNECT_TIMEOUT);
Assert.assertEquals(30000, readTimeOut03);
Assert.assertEquals(30000, connectTimeOut03);
client03.destroy();
}
@Test
public void testJerseyClient() {
// Case1, Set to negative 1.
YarnConfiguration conf = new YarnConfiguration();
conf.setLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, -1L);
conf.setLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, -1L);
int connectTimeOut = (int) getTimeDuration(conf,
YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT);
int readTimeout = (int) getTimeDuration(conf,
YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT);
Assert.assertEquals(-1, connectTimeOut);
Assert.assertEquals(-1, readTimeout);
// Case2, Set the maximum value that exceeds the integer.
// Converted to int, there will be a value out of bounds.
YarnConfiguration conf1 = new YarnConfiguration();
long connectTimeOutLong = (long) Integer.MAX_VALUE + 1;
long readTimeOutLong = (long) Integer.MAX_VALUE + 1;
conf1.setLong(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, connectTimeOutLong);
conf1.setLong(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, readTimeOutLong);
int connectTimeOut1 = (int) getTimeDuration(conf1,
YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT);
int readTimeout1 = (int) getTimeDuration(conf1,
YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT,
YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT);
Assert.assertEquals(-2147483648, connectTimeOut1);
Assert.assertEquals(-2147483648, readTimeout1);
}
private long getTimeDuration(YarnConfiguration conf, String varName, long defaultValue) {
return conf.getTimeDuration(varName, defaultValue, TimeUnit.MILLISECONDS);
}
}