big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16152. Upgrade Eclipse Jetty version to 9.4.x. Contributed by Yuming Wang, Siyao Meng.

Browse Source 
Co-authored-By: Siyao Meng <smeng@cloudera.com>
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
Yuming Wang 2019-10-17 13:09:53 -07:00 committed by Wei-Chiu Chuang
parent f0699a7406
commit 3d41f33018
10 changed files with 67 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-client-modules/hadoop-client-minicluster/pom.xml
View File

@ -811,6 +811,15 @@
<exclude>*/**</exclude>
</excludes>
</filter>
<!-- Jetty 9.4.x: jetty-client and jetty-xml are depended by org.eclipse.jetty.websocket:websocket-client.
But we are only excluding jetty-client not jetty-xml because HttpServer2 implicitly uses the shaded package name.
-->
<filter>
<artifact>org.eclipse.jetty:jetty-client</artifact>
<excludes>
<exclude>*/**</exclude>
</excludes>
</filter>
</filters>
<!-- relocate classes from mssql-jdbc -->
@ -939,6 +948,13 @@
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>javax/websocket/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.websocket.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>jersey/</pattern>
<shadedPattern>${shaded.dependency.prefix}.jersey.</shadedPattern>

3
hadoop-client-modules/hadoop-client-runtime/pom.xml
View File

@ -161,6 +161,9 @@
<exclude>org.eclipse.jetty.websocket:*</exclude>
<exclude>org.eclipse.jetty:jetty-servlet</exclude>
<exclude>org.eclipse.jetty:jetty-security</exclude>
<exclude>org.eclipse.jetty:jetty-client</exclude>
<exclude>org.eclipse.jetty:jetty-http</exclude>
<exclude>org.eclipse.jetty:jetty-xml</exclude>
<exclude>org.ow2.asm:*</exclude>
<!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
<exclude>org.bouncycastle:*</exclude>

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java
View File

@ -24,7 +24,8 @@
import org.apache.commons.logging.LogConfigurationException;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Appender;
import org.eclipse.jetty.server.NCSARequestLog;
import org.eclipse.jetty.server.AsyncRequestLogWriter;
import org.eclipse.jetty.server.CustomRequestLog;
import org.eclipse.jetty.server.RequestLog;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -85,10 +86,11 @@ public static RequestLog getRequestLog(String name) {
if (appender instanceof HttpRequestLogAppender) {
HttpRequestLogAppender requestLogAppender
= (HttpRequestLogAppender)appender;
NCSARequestLog requestLog = new NCSARequestLog();
requestLog.setFilename(requestLogAppender.getFilename());
requestLog.setRetainDays(requestLogAppender.getRetainDays());
return requestLog;
AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter();
logWriter.setFilename(requestLogAppender.getFilename());
logWriter.setRetainDays(requestLogAppender.getRetainDays());
return new CustomRequestLog(logWriter,
CustomRequestLog.EXTENDED_NCSA_FORMAT);
} else {
LOG.warn("Jetty request log for {} was of the wrong class", loggerName);
return null;

44
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
View File

@ -88,13 +88,11 @@
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SessionManager;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.handler.AllowSymLinkAliasChecker;
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.handler.RequestLogHandler;
import org.eclipse.jetty.server.session.AbstractSessionManager;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.FilterMapping;
@ -519,7 +517,8 @@ private ServerConnector createHttpsChannelConnector(
httpConfig.addCustomizer(new SecureRequestCustomizer());
ServerConnector conn = createHttpChannelConnector(server, httpConfig);
SslContextFactory sslContextFactory = new SslContextFactory();
SslContextFactory.Server sslContextFactory =
new SslContextFactory.Server();
sslContextFactory.setNeedClientAuth(needsClientAuth);
sslContextFactory.setKeyManagerPassword(keyPassword);
if (keyStore != null) {
@ -621,12 +620,9 @@ private void initializeWebServer(String name, String hostName,
threadPool.setMaxThreads(maxThreads);
}
SessionManager sm = webAppContext.getSessionHandler().getSessionManager();
if (sm instanceof AbstractSessionManager) {
AbstractSessionManager asm = (AbstractSessionManager)sm;
asm.setHttpOnly(true);
asm.getSessionCookieConfig().setSecure(true);
}
SessionHandler handler = webAppContext.getSessionHandler();
handler.setHttpOnly(true);
handler.getSessionCookieConfig().setSecure(true);
ContextHandlerCollection contexts = new ContextHandlerCollection();
RequestLog requestLog = HttpRequestLog.getRequestLog(name);
@ -777,12 +773,8 @@ protected void addDefaultApps(ContextHandlerCollection parent,
}
logContext.setDisplayName("logs");
SessionHandler handler = new SessionHandler();
SessionManager sm = handler.getSessionManager();
if (sm instanceof AbstractSessionManager) {
AbstractSessionManager asm = (AbstractSessionManager) sm;
asm.setHttpOnly(true);
asm.getSessionCookieConfig().setSecure(true);
}
handler.setHttpOnly(true);
handler.getSessionCookieConfig().setSecure(true);
logContext.setSessionHandler(handler);
logContext.addAliasCheck(new AllowSymLinkAliasChecker());
setContextAttributes(logContext, conf);
@ -800,12 +792,8 @@ protected void addDefaultApps(ContextHandlerCollection parent,
params.put("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
params.put("org.eclipse.jetty.servlet.Default.gzip", "true");
SessionHandler handler = new SessionHandler();
SessionManager sm = handler.getSessionManager();
if (sm instanceof AbstractSessionManager) {
AbstractSessionManager asm = (AbstractSessionManager) sm;
asm.setHttpOnly(true);
asm.getSessionCookieConfig().setSecure(true);
}
handler.setHttpOnly(true);
handler.getSessionCookieConfig().setSecure(true);
staticContext.setSessionHandler(handler);
staticContext.addAliasCheck(new AllowSymLinkAliasChecker());
setContextAttributes(staticContext, conf);
@ -1268,7 +1256,7 @@ private static void bindListener(ServerConnector listener) throws Exception {
* @return
*/
private static BindException constructBindException(ServerConnector listener,
BindException ex) {
IOException ex) {
BindException be = new BindException("Port in use: "
+ listener.getHost() + ":" + listener.getPort());
if (ex != null) {
@ -1290,7 +1278,7 @@ private void bindForSinglePort(ServerConnector listener, int port)
try {
bindListener(listener);
break;
} catch (BindException ex) {
} catch (IOException ex) {
if (port == 0 || !findPort) {
throw constructBindException(listener, ex);
}
@ -1310,13 +1298,13 @@ private void bindForSinglePort(ServerConnector listener, int port)
*/
private void bindForPortRange(ServerConnector listener, int startPort)
throws Exception {
BindException bindException = null;
IOException ioException = null;
try {
bindListener(listener);
return;
} catch (BindException ex) {
} catch (IOException ex) {
// Ignore exception.
bindException = ex;
ioException = ex;
}
for(Integer port : portRanges) {
if (port == startPort) {
@ -1329,10 +1317,10 @@ private void bindForPortRange(ServerConnector listener, int startPort)
return;
} catch (BindException ex) {
// Ignore exception. Move to next port.
bindException = ex;
ioException = ex;
}
}
throw constructBindException(listener, bindException);
throw constructBindException(listener, ioException);
}
/**

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.http;
import org.apache.log4j.Logger;
import org.eclipse.jetty.server.NCSARequestLog;
import org.eclipse.jetty.server.CustomRequestLog;
import org.eclipse.jetty.server.RequestLog;
import org.junit.Test;
@ -42,6 +42,7 @@ public void testAppenderDefined() {
RequestLog requestLog = HttpRequestLog.getRequestLog("test");
Logger.getLogger("http.requests.test").removeAppender(requestLogAppender);
assertNotNull("RequestLog should not be null", requestLog);
assertEquals("Class mismatch", NCSARequestLog.class, requestLog.getClass());
assertEquals("Class mismatch",
CustomRequestLog.class, requestLog.getClass());
}
}

3
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java
View File

@ -105,7 +105,8 @@ private Server createJettyServer() {
conn.setHost(host);
conn.setPort(port);
if (ssl) {
SslContextFactory sslContextFactory = new SslContextFactory();
SslContextFactory.Server sslContextFactory =
new SslContextFactory.Server();
sslContextFactory.setNeedClientAuth(false);
sslContextFactory.setKeyStorePath(keyStore);
sslContextFactory.setKeyStoreType(keyStoreType);

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
View File

@ -83,8 +83,9 @@ public class DatanodeHttpServer implements Closeable {
// set them to the minimum possible
private static final int HTTP_SELECTOR_THREADS = 1;
private static final int HTTP_ACCEPTOR_THREADS = 1;
// Jetty 9.4.x: Adding one more thread to HTTP_MAX_THREADS.
private static final int HTTP_MAX_THREADS =
HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 1;
HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 2;
private final HttpServer2 infoServer;
private final EventLoopGroup bossGroup;
private final EventLoopGroup workerGroup;

2
hadoop-project/pom.xml
View File

@ -35,7 +35,7 @@
<failIfNoTests>false</failIfNoTests>
<maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
<jetty.version>9.3.27.v20190418</jetty.version>
<jetty.version>9.4.20.v20190813</jetty.version>
<test.exclude>_</test.exclude>
<test.exclude.pattern>_</test.exclude.pattern>

3
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
View File

@ -97,7 +97,6 @@
import org.apache.hadoop.yarn.util.UTCClock;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.eclipse.jetty.util.ConcurrentHashSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -371,7 +370,7 @@ private void startNM() throws YarnException, IOException,
// create NM simulators
Random random = new Random();
Set<String> rackSet = new ConcurrentHashSet<>();
Set<String> rackSet = ConcurrentHashMap.newKeySet();
int threadPoolSize = Math.max(poolSize,
SLSConfiguration.RUNNER_POOL_SIZE_DEFAULT);
ExecutorService executorService = Executors.

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml
View File

@ -126,6 +126,14 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-io</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-xml</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-http</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
@ -143,6 +151,14 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-io</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-xml</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-http</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>