diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 918b374137..5641b62e4b 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -811,6 +811,15 @@ */** + + + org.eclipse.jetty:jetty-client + + */** + + @@ -939,6 +948,13 @@ **/pom.xml + + javax/websocket/ + ${shaded.dependency.prefix}.javax.websocket. + + **/pom.xml + + jersey/ ${shaded.dependency.prefix}.jersey. diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index 565e9682d5..552cd9c1d8 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -161,6 +161,9 @@ org.eclipse.jetty.websocket:* org.eclipse.jetty:jetty-servlet org.eclipse.jetty:jetty-security + org.eclipse.jetty:jetty-client + org.eclipse.jetty:jetty-http + org.eclipse.jetty:jetty-xml org.ow2.asm:* org.bouncycastle:* diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java index 05573a8de9..b2f18538b6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java @@ -24,7 +24,8 @@ import org.apache.commons.logging.LogConfigurationException; import org.apache.commons.logging.LogFactory; import org.apache.log4j.Appender; -import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.AsyncRequestLogWriter; +import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,10 +86,11 @@ public static RequestLog getRequestLog(String name) { if (appender instanceof HttpRequestLogAppender) { HttpRequestLogAppender requestLogAppender = (HttpRequestLogAppender)appender; - NCSARequestLog requestLog = new NCSARequestLog(); - requestLog.setFilename(requestLogAppender.getFilename()); - requestLog.setRetainDays(requestLogAppender.getRetainDays()); - return requestLog; + AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter(); + logWriter.setFilename(requestLogAppender.getFilename()); + logWriter.setRetainDays(requestLogAppender.getRetainDays()); + return new CustomRequestLog(logWriter, + CustomRequestLog.EXTENDED_NCSA_FORMAT); } else { LOG.warn("Jetty request log for {} was of the wrong class", loggerName); return null; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 74c81331d7..7e7d64423f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -88,13 +88,11 @@ import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.SessionManager; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.AllowSymLinkAliasChecker; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.server.handler.RequestLogHandler; -import org.eclipse.jetty.server.session.AbstractSessionManager; import org.eclipse.jetty.server.session.SessionHandler; import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.FilterMapping; @@ -519,7 +517,8 @@ private ServerConnector createHttpsChannelConnector( httpConfig.addCustomizer(new SecureRequestCustomizer()); ServerConnector conn = createHttpChannelConnector(server, httpConfig); - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = + new SslContextFactory.Server(); sslContextFactory.setNeedClientAuth(needsClientAuth); sslContextFactory.setKeyManagerPassword(keyPassword); if (keyStore != null) { @@ -621,12 +620,9 @@ private void initializeWebServer(String name, String hostName, threadPool.setMaxThreads(maxThreads); } - SessionManager sm = webAppContext.getSessionHandler().getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager)sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + SessionHandler handler = webAppContext.getSessionHandler(); + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); ContextHandlerCollection contexts = new ContextHandlerCollection(); RequestLog requestLog = HttpRequestLog.getRequestLog(name); @@ -777,12 +773,8 @@ protected void addDefaultApps(ContextHandlerCollection parent, } logContext.setDisplayName("logs"); SessionHandler handler = new SessionHandler(); - SessionManager sm = handler.getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager) sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); logContext.setSessionHandler(handler); logContext.addAliasCheck(new AllowSymLinkAliasChecker()); setContextAttributes(logContext, conf); @@ -800,12 +792,8 @@ protected void addDefaultApps(ContextHandlerCollection parent, params.put("org.eclipse.jetty.servlet.Default.dirAllowed", "false"); params.put("org.eclipse.jetty.servlet.Default.gzip", "true"); SessionHandler handler = new SessionHandler(); - SessionManager sm = handler.getSessionManager(); - if (sm instanceof AbstractSessionManager) { - AbstractSessionManager asm = (AbstractSessionManager) sm; - asm.setHttpOnly(true); - asm.getSessionCookieConfig().setSecure(true); - } + handler.setHttpOnly(true); + handler.getSessionCookieConfig().setSecure(true); staticContext.setSessionHandler(handler); staticContext.addAliasCheck(new AllowSymLinkAliasChecker()); setContextAttributes(staticContext, conf); @@ -1268,7 +1256,7 @@ private static void bindListener(ServerConnector listener) throws Exception { * @return */ private static BindException constructBindException(ServerConnector listener, - BindException ex) { + IOException ex) { BindException be = new BindException("Port in use: " + listener.getHost() + ":" + listener.getPort()); if (ex != null) { @@ -1290,7 +1278,7 @@ private void bindForSinglePort(ServerConnector listener, int port) try { bindListener(listener); break; - } catch (BindException ex) { + } catch (IOException ex) { if (port == 0 || !findPort) { throw constructBindException(listener, ex); } @@ -1310,13 +1298,13 @@ private void bindForSinglePort(ServerConnector listener, int port) */ private void bindForPortRange(ServerConnector listener, int startPort) throws Exception { - BindException bindException = null; + IOException ioException = null; try { bindListener(listener); return; - } catch (BindException ex) { + } catch (IOException ex) { // Ignore exception. - bindException = ex; + ioException = ex; } for(Integer port : portRanges) { if (port == startPort) { @@ -1329,10 +1317,10 @@ private void bindForPortRange(ServerConnector listener, int startPort) return; } catch (BindException ex) { // Ignore exception. Move to next port. - bindException = ex; + ioException = ex; } } - throw constructBindException(listener, bindException); + throw constructBindException(listener, ioException); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java index 212807f78e..d0123e3203 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java @@ -18,7 +18,7 @@ package org.apache.hadoop.http; import org.apache.log4j.Logger; -import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; import org.junit.Test; @@ -42,6 +42,7 @@ public void testAppenderDefined() { RequestLog requestLog = HttpRequestLog.getRequestLog("test"); Logger.getLogger("http.requests.test").removeAppender(requestLogAppender); assertNotNull("RequestLog should not be null", requestLog); - assertEquals("Class mismatch", NCSARequestLog.class, requestLog.getClass()); + assertEquals("Class mismatch", + CustomRequestLog.class, requestLog.getClass()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java index 1da3901b39..e4df118435 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java @@ -105,7 +105,8 @@ private Server createJettyServer() { conn.setHost(host); conn.setPort(port); if (ssl) { - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = + new SslContextFactory.Server(); sslContextFactory.setNeedClientAuth(false); sslContextFactory.setKeyStorePath(keyStore); sslContextFactory.setKeyStoreType(keyStoreType); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java index 86672b403c..e4a0c20976 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java @@ -83,8 +83,9 @@ public class DatanodeHttpServer implements Closeable { // set them to the minimum possible private static final int HTTP_SELECTOR_THREADS = 1; private static final int HTTP_ACCEPTOR_THREADS = 1; + // Jetty 9.4.x: Adding one more thread to HTTP_MAX_THREADS. private static final int HTTP_MAX_THREADS = - HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 1; + HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 2; private final HttpServer2 infoServer; private final EventLoopGroup bossGroup; private final EventLoopGroup workerGroup; diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 9c83560ae6..803cf7cc65 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -35,7 +35,7 @@ false true - 9.3.27.v20190418 + 9.4.20.v20190813 _ _ diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index f99038ebe5..6f75bd17c6 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -97,7 +97,6 @@ import org.apache.hadoop.yarn.util.UTCClock; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; -import org.eclipse.jetty.util.ConcurrentHashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -371,7 +370,7 @@ private void startNM() throws YarnException, IOException, // create NM simulators Random random = new Random(); - Set rackSet = new ConcurrentHashSet<>(); + Set rackSet = ConcurrentHashMap.newKeySet(); int threadPoolSize = Math.max(poolSize, SLSConfiguration.RUNNER_POOL_SIZE_DEFAULT); ExecutorService executorService = Executors. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml index 35718da4d2..0924f6e59d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml @@ -126,6 +126,14 @@ org.eclipse.jetty jetty-io + + org.eclipse.jetty + jetty-xml + + + org.eclipse.jetty + jetty-http + test @@ -143,6 +151,14 @@ org.eclipse.jetty jetty-io + + org.eclipse.jetty + jetty-xml + + + org.eclipse.jetty + jetty-http + test