diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index df482c1859..c1bb6aa68d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2672,6 +2672,20 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_RM_APPLICATION_HTTPS_POLICY = "NONE"; + + // If the proxy connection time enabled. + public static final String RM_PROXY_TIMEOUT_ENABLED = + RM_PREFIX + "proxy.timeout.enabled"; + + public static final boolean DEFALUT_RM_PROXY_TIMEOUT_ENABLED = + true; + + public static final String RM_PROXY_CONNECTION_TIMEOUT = + RM_PREFIX + "proxy.connection.timeout"; + + public static final int DEFAULT_RM_PROXY_CONNECTION_TIMEOUT = + 60000; + /** * Interval of time the linux container executor should try cleaning up * cgroups entry when cleaning up a container. This is required due to what diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index ff3a817913..4be357b78a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2601,6 +2601,18 @@ + + Enable the web proxy connection timeout, default is enabled. + yarn.resourcemanager.proxy.timeout.enabled + true + + + + The web proxy connection timeout. + yarn.resourcemanager.proxy.connection.timeout + 60000 + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index 0b6bb65d8d..03b7077bc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -122,6 +122,9 @@ public HTML html() { } } + protected void setConf(YarnConfiguration conf){ + this.conf = conf; + } /** * Default constructor */ @@ -230,6 +233,14 @@ private void proxyLink(final HttpServletRequest req, String httpsPolicy = conf.get(YarnConfiguration.RM_APPLICATION_HTTPS_POLICY, YarnConfiguration.DEFAULT_RM_APPLICATION_HTTPS_POLICY); + + boolean connectionTimeoutEnabled = + conf.getBoolean(YarnConfiguration.RM_PROXY_TIMEOUT_ENABLED, + YarnConfiguration.DEFALUT_RM_PROXY_TIMEOUT_ENABLED); + int connectionTimeout = + conf.getInt(YarnConfiguration.RM_PROXY_CONNECTION_TIMEOUT, + YarnConfiguration.DEFAULT_RM_PROXY_CONNECTION_TIMEOUT); + if (httpsPolicy.equals("LENIENT") || httpsPolicy.equals("STRICT")) { ProxyCA proxyCA = getProxyCA(); // ProxyCA could be null when the Proxy is run outside the RM @@ -250,10 +261,18 @@ private void proxyLink(final HttpServletRequest req, InetAddress localAddress = InetAddress.getByName(proxyHost); LOG.debug("local InetAddress for proxy host: {}", localAddress); httpClientBuilder.setDefaultRequestConfig( - RequestConfig.custom() - .setCircularRedirectsAllowed(true) - .setLocalAddress(localAddress) - .build()); + connectionTimeoutEnabled ? + RequestConfig.custom() + .setCircularRedirectsAllowed(true) + .setLocalAddress(localAddress) + .setConnectionRequestTimeout(connectionTimeout) + .setSocketTimeout(connectionTimeout) + .setConnectTimeout(connectionTimeout) + .build() : + RequestConfig.custom() + .setCircularRedirectsAllowed(true) + .setLocalAddress(localAddress) + .build()); HttpRequestBase base = null; if (method.equals(HTTP.GET)) { @@ -621,7 +640,6 @@ private FetchedAppReport getFetchedAppReport(ApplicationId id) * again... If this method returns true, there was a redirect, and * it was handled by redirecting the current request to an error page. * - * @param path the part of the request path after the app id * @param id the app id * @param req the request object * @param resp the response object diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index f05e05a2d6..6c8993f6e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -23,6 +23,8 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -35,10 +37,14 @@ import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; +import java.net.SocketTimeoutException; +import java.util.Collections; import java.util.Enumeration; import java.util.List; import java.util.Map; +import javax.servlet.ServletConfig; +import javax.servlet.ServletContext; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; @@ -98,6 +104,7 @@ public static void start() throws Exception { context.setContextPath("/foo"); server.setHandler(context); context.addServlet(new ServletHolder(TestServlet.class), "/bar"); + context.addServlet(new ServletHolder(TimeOutTestServlet.class), "/timeout"); ((ServerConnector)server.getConnectors()[0]).setHost("localhost"); server.start(); originalPort = ((ServerConnector)server.getConnectors()[0]).getLocalPort(); @@ -145,6 +152,29 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) } } + @SuppressWarnings("serial") + public static class TimeOutTestServlet extends HttpServlet { + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + LOG.warn("doGet() interrupted", e); + resp.setStatus(HttpServletResponse.SC_BAD_REQUEST); + return; + } + resp.setStatus(HttpServletResponse.SC_OK); + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + resp.setStatus(HttpServletResponse.SC_OK); + } + } + @Test(timeout=5000) public void testWebAppProxyServlet() throws Exception { configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); @@ -256,6 +286,45 @@ public void testWebAppProxyServlet() throws Exception { } } + @Test(expected = SocketTimeoutException.class) + public void testWebAppProxyConnectionTimeout() + throws IOException, ServletException{ + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRemoteUser()).thenReturn("dr.who"); + when(request.getPathInfo()).thenReturn("/application_00_0"); + when(request.getHeaderNames()).thenReturn(Collections.emptyEnumeration()); + + HttpServletResponse response = mock(HttpServletResponse.class); + when(response.getOutputStream()).thenReturn(null); + + WebAppProxyServlet servlet = new WebAppProxyServlet(); + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.RM_PROXY_TIMEOUT_ENABLED, + true); + conf.setInt(YarnConfiguration.RM_PROXY_CONNECTION_TIMEOUT, + 1000); + + servlet.setConf(conf); + + ServletConfig config = mock(ServletConfig.class); + ServletContext context = mock(ServletContext.class); + when(config.getServletContext()).thenReturn(context); + + AppReportFetcherForTest appReportFetcher = + new AppReportFetcherForTest(new YarnConfiguration()); + + when(config.getServletContext() + .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE)) + .thenReturn(appReportFetcher); + + appReportFetcher.answer = 7; + + servlet.init(config); + servlet.doGet(request, response); + + } + @Test(timeout=5000) public void testAppReportForEmptyTrackingUrl() throws Exception { configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); @@ -391,9 +460,9 @@ public void testWebAppProxyServerMainMethod() throws Exception { @Test(timeout=5000) public void testCheckHttpsStrictAndNotProvided() throws Exception { - HttpServletResponse resp = Mockito.mock(HttpServletResponse.class); + HttpServletResponse resp = mock(HttpServletResponse.class); StringWriter sw = new StringWriter(); - Mockito.when(resp.getWriter()).thenReturn(new PrintWriter(sw)); + when(resp.getWriter()).thenReturn(new PrintWriter(sw)); YarnConfiguration conf = new YarnConfiguration(); final URI httpLink = new URI("http://foo.com"); final URI httpsLink = new URI("https://foo.com"); @@ -566,6 +635,12 @@ public FetchedAppReport getApplicationReport(ApplicationId appId) return result; } else if (answer == 6) { return getDefaultApplicationReport(appId, false); + } else if (answer == 7) { + // test connection timeout + FetchedAppReport result = getDefaultApplicationReport(appId); + result.getApplicationReport().setOriginalTrackingUrl("localhost:" + + originalPort + "/foo/timeout?a=b#main"); + return result; } return null; }