From e780556ae9229fe7a90817eb4e5449d7eed35dd8 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 5 Sep 2018 09:52:35 -0700 Subject: [PATCH] HADOOP-15696. KMS performance regression due to too many open file descriptors after Jetty migration. Contributed by Wei-Chiu Chuang. --- .../org/apache/hadoop/http/HttpServer2.java | 26 ++++++++++--------- .../apache/hadoop/http/TestHttpServer.java | 14 ++++++++++ .../src/main/resources/kms-default.xml | 7 +++++ .../src/main/resources/httpfs-default.xml | 7 +++++ 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 2435671a31..d2ba469ab4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -141,6 +141,10 @@ public final class HttpServer2 implements FilterContainer { "hadoop.http.selector.count"; // -1 to use default behavior of setting count based on CPU core count public static final int HTTP_SELECTOR_COUNT_DEFAULT = -1; + // idle timeout in milliseconds + public static final String HTTP_IDLE_TIMEOUT_MS_KEY = + "hadoop.http.idle_timeout.ms"; + public static final int HTTP_IDLE_TIMEOUT_MS_DEFAULT = 10000; public static final String HTTP_TEMP_DIR_KEY = "hadoop.http.temp.dir"; public static final String FILTER_INITIALIZER_PROPERTY @@ -445,6 +449,8 @@ public HttpServer2 build() throws IOException { int responseHeaderSize = conf.getInt( HTTP_MAX_RESPONSE_HEADER_SIZE_KEY, HTTP_MAX_RESPONSE_HEADER_SIZE_DEFAULT); + int idleTimeout = conf.getInt(HTTP_IDLE_TIMEOUT_MS_KEY, + HTTP_IDLE_TIMEOUT_MS_DEFAULT); HttpConfiguration httpConfig = new HttpConfiguration(); httpConfig.setRequestHeaderSize(requestHeaderSize); @@ -470,6 +476,7 @@ public HttpServer2 build() throws IOException { connector.setHost(ep.getHost()); connector.setPort(ep.getPort() == -1 ? 0 : ep.getPort()); connector.setAcceptQueueSize(backlogSize); + connector.setIdleTimeout(idleTimeout); server.addListener(connector); } server.loadListeners(); @@ -483,7 +490,13 @@ private ServerConnector createHttpChannelConnector( conf.getInt(HTTP_SELECTOR_COUNT_KEY, HTTP_SELECTOR_COUNT_DEFAULT)); ConnectionFactory connFactory = new HttpConnectionFactory(httpConfig); conn.addConnectionFactory(connFactory); - configureChannelConnector(conn); + if(Shell.WINDOWS) { + // result of setting the SO_REUSEADDR flag is different on Windows + // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx + // without this 2 NN's can start on the same machine and listen on + // the same port with indeterminate routing of incoming requests to them + conn.setReuseAddress(false); + } return conn; } @@ -659,17 +672,6 @@ private static void addNoCacheFilter(ServletContextHandler ctxt) { Collections. emptyMap(), new String[] { "/*" }); } - private static void configureChannelConnector(ServerConnector c) { - c.setIdleTimeout(10000); - if(Shell.WINDOWS) { - // result of setting the SO_REUSEADDR flag is different on Windows - // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx - // without this 2 NN's can start on the same machine and listen on - // the same port with indeterminate routing of incoming requests to them - c.setReuseAddress(false); - } - } - /** Get an array of FilterConfiguration specified in the conf */ private static FilterInitializer[] getFilterInitializers(Configuration conf) { if (conf == null) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index 26b1137e49..e0c87e93a9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -55,6 +55,7 @@ import javax.ws.rs.core.MediaType; import java.io.IOException; import java.io.PrintWriter; +import java.lang.reflect.Field; import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; @@ -702,6 +703,19 @@ public void testBacklogSize() throws Exception assertEquals(backlogSize, listener.getAcceptQueueSize()); } + @Test + public void testIdleTimeout() throws Exception { + final int idleTimeout = 1000; + Configuration conf = new Configuration(); + conf.setInt(HttpServer2.HTTP_IDLE_TIMEOUT_MS_KEY, idleTimeout); + HttpServer2 srv = createServer("test", conf); + Field f = HttpServer2.class.getDeclaredField("listeners"); + f.setAccessible(true); + List listeners = (List) f.get(srv); + ServerConnector listener = (ServerConnector)listeners.get(0); + assertEquals(idleTimeout, listener.getIdleTimeout()); + } + @Test public void testHttpResponseDefaultHeaders() throws Exception { Configuration conf = new Configuration(); diff --git a/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml b/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml index 434adcbc8a..783f4e6c03 100644 --- a/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml +++ b/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml @@ -101,6 +101,13 @@ + + hadoop.http.idle_timeout.ms + 1000 + + KMS Server connection timeout in milliseconds. + + diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml index e4204564c2..3e9064f447 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml @@ -54,6 +54,13 @@ + + hadoop.http.idle_timeout.ms + 1000 + + Httpfs Server connection timeout in milliseconds. + + hadoop.http.max.threads