big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16371: Option to disable GCM for SSL connections when running on Java 8.

Browse Source 
Contributed by Sahil Takiar.

This moves the SSLSocketFactoryEx class from hadoop-azure into hadoop-common
as the DelegatingSSLSocketFactory and binds the S3A connector to it so that
it can avoid using those HTTPS algorithms which are underperformant on Java 8.

Change-Id: Ie9e6ac24deac1aa05e136e08899620efa7d22abd
This commit is contained in:
Sahil Takiar 2019-09-17 11:30:18 +01:00 committed by Steve Loughran
parent 3a549cea19
commit 55ce454ce4
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
15 changed files with 397 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-common-project/hadoop-common/pom.xml
View File

@ -343,6 +343,16 @@
<artifactId>dnsjava</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.wildfly.openssl</groupId>
<artifactId>wildfly-openssl</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

117
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/SSLSocketFactoryEx.java → hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.fs.azurebfs.utils;
package org.apache.hadoop.security.ssl;
import java.io.IOException;
import java.net.InetAddress;
@ -38,30 +38,60 @@
/**
* Extension to use native OpenSSL library instead of JSSE for better
* performance.
* A {@link SSLSocketFactory} that can delegate to various SSL implementations.
* Specifically, either OpenSSL or JSSE can be used. OpenSSL offers better
* performance than JSSE and is made available via the
* <a href="https://github.com/wildfly/wildfly-openssl">wildlfy-openssl</a>
* library.
*
* <p>
* The factory has several different modes of operation:
* <ul>
* <li>OpenSSL: Uses the wildly-openssl library to delegate to the
* system installed OpenSSL. If the wildfly-openssl integration is not
* properly setup, an exception is thrown.</li>
* <li>Default: Attempts to use the OpenSSL mode, if it cannot load the
* necessary libraries, it falls back to the Default_JSEE mode.</li>
* <li>Default_JSSE: Delegates to the JSSE implementation of SSL, but
* it disables the GCM cipher when running on Java 8.</li>
* <li>Default_JSSE_with_GCM: Delegates to the JSSE implementation of
* SSL with no modification to the list of enabled ciphers.</li>
* </ul>
* </p>
*/
public final class SSLSocketFactoryEx extends SSLSocketFactory {
public final class DelegatingSSLSocketFactory extends SSLSocketFactory {
/**
* Default indicates Ordered, preferred OpenSSL, if failed to load then fall
* back to Default_JSSE
* back to Default_JSSE.
*
* <p>
* Default_JSSE is not truly the the default JSSE implementation because
* the GCM cipher is disabled when running on Java 8. However, the name
* was not changed in order to preserve backwards compatibility. Instead,
* a new mode called Default_JSSE_with_GCM delegates to the default JSSE
* implementation with no changes to the list of enabled ciphers.
* </p>
*/
public enum SSLChannelMode {
OpenSSL,
Default,
Default_JSSE
Default_JSSE,
Default_JSSE_with_GCM
}
private static SSLSocketFactoryEx instance = null;
private static DelegatingSSLSocketFactory instance = null;
private static final Logger LOG = LoggerFactory.getLogger(
SSLSocketFactoryEx.class);
DelegatingSSLSocketFactory.class);
private String providerName;
private SSLContext ctx;
private String[] ciphers;
private SSLChannelMode channelMode;
// This should only be modified within the #initializeDefaultFactory
// method which is synchronized
private boolean openSSLProviderRegistered;
/**
* Initialize a singleton SSL socket factory.
*
@ -71,7 +101,7 @@ public enum SSLChannelMode {
public static synchronized void initializeDefaultFactory(
SSLChannelMode preferredMode) throws IOException {
if (instance == null) {
instance = new SSLSocketFactoryEx(preferredMode);
instance = new DelegatingSSLSocketFactory(preferredMode);
}
}
@ -84,15 +114,11 @@ public static synchronized void initializeDefaultFactory(
* @return instance of the SSLSocketFactory, instance must be initialized by
* initializeDefaultFactory.
*/
public static SSLSocketFactoryEx getDefaultFactory() {
public static DelegatingSSLSocketFactory getDefaultFactory() {
return instance;
}
static {
OpenSSLProvider.register();
}
private SSLSocketFactoryEx(SSLChannelMode preferredChannelMode)
private DelegatingSSLSocketFactory(SSLChannelMode preferredChannelMode)
throws IOException {
try {
initializeSSLContext(preferredChannelMode);
@ -118,33 +144,47 @@ private SSLSocketFactoryEx(SSLChannelMode preferredChannelMode)
private void initializeSSLContext(SSLChannelMode preferredChannelMode)
throws NoSuchAlgorithmException, KeyManagementException {
switch (preferredChannelMode) {
case Default:
try {
java.util.logging.Logger logger = java.util.logging.Logger.getLogger(SSL.class.getName());
logger.setLevel(Level.WARNING);
ctx = SSLContext.getInstance("openssl.TLS");
ctx.init(null, null, null);
// Strong reference needs to be kept to logger until initialization of SSLContext finished (see HADOOP-16174):
logger.setLevel(Level.INFO);
channelMode = SSLChannelMode.OpenSSL;
} catch (NoSuchAlgorithmException e) {
LOG.warn("Failed to load OpenSSL. Falling back to the JSSE default.");
ctx = SSLContext.getDefault();
channelMode = SSLChannelMode.Default_JSSE;
}
break;
case OpenSSL:
case Default:
if (!openSSLProviderRegistered) {
OpenSSLProvider.register();
openSSLProviderRegistered = true;
}
try {
java.util.logging.Logger logger = java.util.logging.Logger.getLogger(
SSL.class.getName());
logger.setLevel(Level.WARNING);
ctx = SSLContext.getInstance("openssl.TLS");
ctx.init(null, null, null);
// Strong reference needs to be kept to logger until initialization of
// SSLContext finished (see HADOOP-16174):
logger.setLevel(Level.INFO);
channelMode = SSLChannelMode.OpenSSL;
break;
case Default_JSSE:
} catch (NoSuchAlgorithmException e) {
LOG.debug("Failed to load OpenSSL. Falling back to the JSSE default.");
ctx = SSLContext.getDefault();
channelMode = SSLChannelMode.Default_JSSE;
break;
default:
throw new AssertionError("Unknown channel mode: "
+ preferredChannelMode);
}
break;
case OpenSSL:
if (!openSSLProviderRegistered) {
OpenSSLProvider.register();
openSSLProviderRegistered = true;
}
ctx = SSLContext.getInstance("openssl.TLS");
ctx.init(null, null, null);
channelMode = SSLChannelMode.OpenSSL;
break;
case Default_JSSE:
ctx = SSLContext.getDefault();
channelMode = SSLChannelMode.Default_JSSE;
break;
case Default_JSSE_with_GCM:
ctx = SSLContext.getDefault();
channelMode = SSLChannelMode.Default_JSSE_with_GCM;
break;
default:
throw new NoSuchAlgorithmException("Unknown channel mode: "
+ preferredChannelMode);
}
}
@ -234,7 +274,8 @@ private String[] alterCipherList(String[] defaultCiphers) {
// Remove GCM mode based ciphers from the supported list.
for (int i = 0; i < defaultCiphers.length; i++) {
if (defaultCiphers[i].contains("_GCM_")) {
LOG.debug("Removed Cipher - " + defaultCiphers[i]);
LOG.debug("Removed Cipher - {} from list of enabled SSLSocket ciphers",
defaultCiphers[i]);
} else {
preferredSuits.add(defaultCiphers[i]);
}

14
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -1973,6 +1973,20 @@
</description>
</property>
<property>
<name>fs.s3a.ssl.channel.mode</name>
<value>default_jsse</value>
<description>
If secure connections to S3 are enabled, configures the SSL
implementation used to encrypt connections to S3. Supported values are:
"default_jsse" and "default_jsse_with_gcm". "default_jsse" uses the Java
Secure Socket Extension package (JSSE). However, when running on Java 8,
the GCM cipher is removed from the list of enabled ciphers. This is due
to performance issues with GCM in Java 8. "default_jsse_with_gcm" uses
the JSSE with the default list of cipher suites.
</description>
</property>
<!-- Azure file system properties -->
<property>
<name>fs.AbstractFileSystem.wasb.impl</name>

57
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestDelegatingSSLSocketFactory.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.security.ssl;
import java.io.IOException;
import java.util.Arrays;
import org.junit.Test;
import org.apache.hadoop.util.NativeCodeLoader;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assume.assumeTrue;
/**
* Tests for {@link DelegatingSSLSocketFactory}.
*/
public class TestDelegatingSSLSocketFactory {
@Test
public void testOpenSSL() throws IOException {
assumeTrue("Unable to load native libraries",
NativeCodeLoader.isNativeCodeLoaded());
assumeTrue("Build was not compiled with support for OpenSSL",
NativeCodeLoader.buildSupportsOpenssl());
DelegatingSSLSocketFactory.initializeDefaultFactory(
DelegatingSSLSocketFactory.SSLChannelMode.OpenSSL);
assertThat(DelegatingSSLSocketFactory.getDefaultFactory()
.getProviderName()).contains("openssl");
}
@Test
public void testJSEENoGCMJava8() throws IOException {
assumeTrue("Not running on Java 8",
System.getProperty("java.version").startsWith("1.8"));
DelegatingSSLSocketFactory.initializeDefaultFactory(
DelegatingSSLSocketFactory.SSLChannelMode.Default_JSSE);
assertThat(Arrays.stream(DelegatingSSLSocketFactory.getDefaultFactory()
.getSupportedCipherSuites())).noneMatch("GCM"::contains);
}
}

7
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View File

@ -20,6 +20,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import java.util.concurrent.TimeUnit;
@ -153,6 +154,12 @@ private Constants() {
"fs.s3a.connection.ssl.enabled";
public static final boolean DEFAULT_SECURE_CONNECTIONS = true;
// use OpenSSL or JSEE for secure connections
public static final String SSL_CHANNEL_MODE = "fs.s3a.ssl.channel.mode";
public static final DelegatingSSLSocketFactory.SSLChannelMode
DEFAULT_SSL_CHANNEL_MODE =
DelegatingSSLSocketFactory.SSLChannelMode.Default_JSSE;
//use a custom endpoint?
public static final String ENDPOINT = "fs.s3a.endpoint";

31
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
View File

@ -48,6 +48,7 @@
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
import org.apache.hadoop.fs.s3a.impl.NetworkBinding;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.ProviderUtils;
@ -1218,14 +1219,15 @@ public static ClientConfiguration createAwsConf(Configuration conf,
*
* @param conf Hadoop configuration
* @param awsConf AWS SDK configuration
*
* @throws IOException if there was an error initializing the protocol
* settings
*/
public static void initConnectionSettings(Configuration conf,
ClientConfiguration awsConf) {
ClientConfiguration awsConf) throws IOException {
awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
DEFAULT_MAXIMUM_CONNECTIONS, 1));
boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
DEFAULT_SECURE_CONNECTIONS);
awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
initProtocolSettings(conf, awsConf);
awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
DEFAULT_MAX_ERROR_RETRIES, 0));
awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
@ -1244,6 +1246,27 @@ public static void initConnectionSettings(Configuration conf,
}
}
/**
* Initializes the connection protocol settings when connecting to S3 (e.g.
* either HTTP or HTTPS). If secure connections are enabled, this method
* will load the configured SSL providers.
*
* @param conf Hadoop configuration
* @param awsConf AWS SDK configuration
*
* @throws IOException if there is an error initializing the configured
* {@link javax.net.ssl.SSLSocketFactory}
*/
private static void initProtocolSettings(Configuration conf,
ClientConfiguration awsConf) throws IOException {
boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
DEFAULT_SECURE_CONNECTIONS);
awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
if (secureConnections) {
NetworkBinding.bindSSLChannelMode(conf, awsConf);
}
}
/**
* Initializes AWS SDK proxy support in the AWS client configuration
* if the S3A settings enable it.

113
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java Normal file
View File

@ -0,0 +1,113 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.s3a.impl;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLSocketFactory;
import com.amazonaws.ClientConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SSL_CHANNEL_MODE;
import static org.apache.hadoop.fs.s3a.Constants.SSL_CHANNEL_MODE;
/**
* Configures network settings when communicating with AWS services.
*/
public class NetworkBinding {
private static final Logger LOG =
LoggerFactory.getLogger(NetworkBinding.class);
private static final String AWS_SOCKET_FACTORY_CLASSNAME = "com.amazonaws" +
".thirdparty.apache.http.conn.ssl.SSLConnectionSocketFactory";
/**
* Configures the {@link com.amazonaws.thirdparty.apache.http.conn.ssl
* .SSLConnectionSocketFactory} used by the AWS SDK. A custom
* SSLConnectionSocketFactory can be set using the method
* {@link com.amazonaws.ApacheHttpClientConfig#setSslSocketFactory(
* com.amazonaws.thirdparty.apache.http.conn.socket.ConnectionSocketFactory)}.
* If {@link com.amazonaws.thirdparty.apache.http.conn.ssl
* .SSLConnectionSocketFactory} cannot be found on the classpath, the value
* of {@link org.apache.hadoop.fs.s3a.Constants#SSL_CHANNEL_MODE} is ignored.
*
* @param conf the {@link Configuration} used to get the client specified
* value of {@link org.apache.hadoop.fs.s3a.Constants
* #SSL_CHANNEL_MODE}
* @param awsConf the {@link ClientConfiguration} to set the
* SSLConnectionSocketFactory for.
* @throws IOException if there is an error while initializing the
* {@link SSLSocketFactory}.
*/
public static void bindSSLChannelMode(Configuration conf,
ClientConfiguration awsConf) throws IOException {
try {
// Validate that SSL_CHANNEL_MODE is set to a valid value.
String channelModeString = conf.get(
SSL_CHANNEL_MODE, DEFAULT_SSL_CHANNEL_MODE.name());
DelegatingSSLSocketFactory.SSLChannelMode channelMode = null;
for (DelegatingSSLSocketFactory.SSLChannelMode mode :
DelegatingSSLSocketFactory.SSLChannelMode.values()) {
if (mode.name().equalsIgnoreCase(channelModeString)) {
channelMode = mode;
}
}
if (channelMode == null) {
throw new IllegalArgumentException(channelModeString +
" is not a valid value for " + SSL_CHANNEL_MODE);
}
if (channelMode == DelegatingSSLSocketFactory.SSLChannelMode.OpenSSL ||
channelMode == DelegatingSSLSocketFactory.SSLChannelMode.Default) {
throw new UnsupportedOperationException("S3A does not support " +
"setting " + SSL_CHANNEL_MODE + " " +
DelegatingSSLSocketFactory.SSLChannelMode.OpenSSL + " or " +
DelegatingSSLSocketFactory.SSLChannelMode.Default);
}
// Look for AWS_SOCKET_FACTORY_CLASSNAME on the classpath and instantiate
// an instance using the DelegatingSSLSocketFactory as the
// SSLSocketFactory.
Class<?> sslConnectionSocketFactory = Class.forName(
AWS_SOCKET_FACTORY_CLASSNAME);
Constructor<?> factoryConstructor =
sslConnectionSocketFactory.getDeclaredConstructor(
SSLSocketFactory.class, HostnameVerifier.class);
DelegatingSSLSocketFactory.initializeDefaultFactory(channelMode);
awsConf.getApacheHttpClientConfig().setSslSocketFactory(
(com.amazonaws.thirdparty.apache.http.conn.ssl.
SSLConnectionSocketFactory) factoryConstructor
.newInstance(DelegatingSSLSocketFactory
.getDefaultFactory(),
(HostnameVerifier) null));
} catch (ClassNotFoundException | NoSuchMethodException |
IllegalAccessException | InstantiationException |
InvocationTargetException e) {
LOG.debug("Unable to create class {}, value of {} will be ignored",
AWS_SOCKET_FACTORY_CLASSNAME, SSL_CHANNEL_MODE, e);
}
}
}

51
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
View File

@ -516,3 +516,54 @@ With an object store this is slow, and may cause problems if the caller
expects an immediate response. For example, a thread may block so long
that other liveness checks start to fail.
Consider spawning off an executor thread to do these background cleanup operations.
## <a name="coding"></a> Tuning SSL Performance
By default, S3A uses HTTPS to communicate with AWS Services. This means that all
communication with S3 is encrypted using SSL. The overhead of this encryption
can significantly slow down applications. The configuration option
`fs.s3a.ssl.channel.mode` allows applications to trigger certain SSL
optimizations.
By default, `fs.s3a.ssl.channel.mode` is set to `default_jsse`, which uses
the Java Secure Socket Extension implementation of SSL (this is the default
implementation when running Java). However, there is one difference, the GCM
cipher is removed from the list of enabled cipher suites when running on Java 8.
The GCM cipher has known performance issues when running on Java 8, see
HADOOP-15669 and HADOOP-16050 for details. It is important to note that the
GCM cipher is only disabled on Java 8. GCM performance has been improved
in Java 9, so if `default_jsse` is specified and applications run on Java
9, they should see no difference compared to running with the vanilla JSSE.
Other options for `fs.s3a.ssl.channel.mode` include `default_jsse_with_gcm`.
This option includes GCM in the list of cipher suites on Java 8, so it is
equivalent to running with the vanilla JSSE. The naming convention is setup
in order to preserve backwards compatibility with HADOOP-15669.
`fs.s3a.ssl.channel.mode` can be configured as follows:
```xml
<property>
<name>fs.s3a.ssl.channel.mode</name>
<value>default_jsse</value>
<description>
If secure connections to S3 are enabled, configures the SSL
implementation used to encrypt connections to S3. Supported values are:
"default_jsse" and "default_jsse_with_gcm". "default_jsse" uses the Java
Secure Socket Extension package (JSSE). However, when running on Java 8,
the GCM cipher is removed from the list of enabled ciphers. This is due
to performance issues with GCM in Java 8. "default_jsse_with_gcm" uses
the JSSE with the default list of cipher suites.
</description>
</property>
```
Supported values for `fs.s3a.ssl.channel.mode`:
| fs.s3a.ssl.channel.mode Value | Description |
|-------------------------------|-------------|
| default_jsse | Uses Java JSSE without GCM on Java 8 |
| default_jsse_with_gcm | Uses Java JSSE |
Other options may be added to `fs.s3a.ssl.channel.mode` in the future as
further SSL optimizations are made.

22
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
View File

@ -40,6 +40,7 @@
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AInputPolicy;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADVISE;
@ -47,8 +48,14 @@
import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADV_RANDOM;
import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADV_SEQUENTIAL;
import static org.apache.hadoop.fs.s3a.Constants.READAHEAD_RANGE;
import static org.apache.hadoop.fs.s3a.Constants.SSL_CHANNEL_MODE;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.FS_S3A_IMPL_DISABLE_CACHE;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
import static org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory.
SSLChannelMode.Default_JSSE;
import static org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory.
SSLChannelMode.Default_JSSE_with_GCM;
/**
* S3A contract tests covering file seek.
@ -62,6 +69,7 @@ public class ITestS3AContractSeek extends AbstractContractSeekTest {
protected static final int READAHEAD = 1024;
private final String seekPolicy;
private final DelegatingSSLSocketFactory.SSLChannelMode sslChannelMode;
public static final int DATASET_LEN = READAHEAD * 2;
@ -75,9 +83,9 @@ public class ITestS3AContractSeek extends AbstractContractSeekTest {
@Parameterized.Parameters
public static Collection<Object[]> params() {
return Arrays.asList(new Object[][]{
{INPUT_FADV_RANDOM},
{INPUT_FADV_NORMAL},
{INPUT_FADV_SEQUENTIAL},
{INPUT_FADV_SEQUENTIAL, Default_JSSE},
{INPUT_FADV_RANDOM, Default_JSSE_with_GCM},
{INPUT_FADV_NORMAL, Default_JSSE_with_GCM},
});
}
@ -85,8 +93,10 @@ public static Collection<Object[]> params() {
* Run the test with a chosen seek policy.
* @param seekPolicy fadvise policy to use.
*/
public ITestS3AContractSeek(final String seekPolicy) {
public ITestS3AContractSeek(final String seekPolicy,
final DelegatingSSLSocketFactory.SSLChannelMode sslChannelMode) {
this.seekPolicy = seekPolicy;
this.sslChannelMode = sslChannelMode;
}
/**
@ -106,7 +116,8 @@ protected Configuration createConfiguration() {
URI bucketURI = new URI(checkNotNull(conf.get("fs.contract.test.fs.s3a")));
S3ATestUtils.removeBucketOverrides(bucketURI.getHost(), conf,
READAHEAD_RANGE,
INPUT_FADVISE);
INPUT_FADVISE,
SSL_CHANNEL_MODE);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
@ -114,6 +125,7 @@ protected Configuration createConfiguration() {
S3ATestUtils.disableFilesystemCaching(conf);
conf.setInt(READAHEAD_RANGE, READAHEAD);
conf.set(INPUT_FADVISE, seekPolicy);
conf.set(SSL_CHANNEL_MODE, sslChannelMode.name());
return conf;
}

4
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
View File

@ -56,7 +56,7 @@
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azurebfs.services.KeyProvider;
import org.apache.hadoop.fs.azurebfs.services.SimpleKeyProvider;
import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.util.ReflectionUtils;
@ -435,7 +435,7 @@ public String getCustomUserAgentPrefix() {
return this.userAgentId;
}
public SSLSocketFactoryEx.SSLChannelMode getPreferredSSLFactoryOption() {
public DelegatingSSLSocketFactory.SSLChannelMode getPreferredSSLFactoryOption() {
return getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE);
}

6
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
View File

@ -20,7 +20,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
/**
* Responsible to keep all the Azure Blob File System related configurations.
@ -59,8 +59,8 @@ public final class FileSystemConfigurations {
public static final boolean DEFAULT_ENABLE_FLUSH = true;
public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = true;
public static final SSLSocketFactoryEx.SSLChannelMode DEFAULT_FS_AZURE_SSL_CHANNEL_MODE
= SSLSocketFactoryEx.SSLChannelMode.Default;
public static final DelegatingSSLSocketFactory.SSLChannelMode DEFAULT_FS_AZURE_SSL_CHANNEL_MODE
= DelegatingSSLSocketFactory.SSLChannelMode.Default;
public static final boolean DEFAULT_ENABLE_DELEGATION_TOKEN = false;
public static final boolean DEFAULT_ENABLE_HTTPS = true;

8
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
View File

@ -29,7 +29,7 @@
import java.util.Locale;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
import org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams;
@ -79,10 +79,10 @@ public AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredent
if (this.baseUrl.toString().startsWith(HTTPS_SCHEME)) {
try {
SSLSocketFactoryEx.initializeDefaultFactory(this.abfsConfiguration.getPreferredSSLFactoryOption());
sslProviderName = SSLSocketFactoryEx.getDefaultFactory().getProviderName();
DelegatingSSLSocketFactory.initializeDefaultFactory(this.abfsConfiguration.getPreferredSSLFactoryOption());
sslProviderName = DelegatingSSLSocketFactory.getDefaultFactory().getProviderName();
} catch (IOException e) {
// Suppress exception. Failure to init SSLSocketFactoryEx would have only performance impact.
// Suppress exception. Failure to init DelegatingSSLSocketFactory would have only performance impact.
}
}

4
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
View File

@ -29,7 +29,7 @@
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSocketFactory;
import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonToken;
@ -180,7 +180,7 @@ public AbfsHttpOperation(final URL url, final String method, final List<AbfsHttp
this.connection = openConnection();
if (this.connection instanceof HttpsURLConnection) {
HttpsURLConnection secureConn = (HttpsURLConnection) this.connection;
SSLSocketFactory sslSocketFactory = SSLSocketFactoryEx.getDefaultFactory();
SSLSocketFactory sslSocketFactory = DelegatingSSLSocketFactory.getDefaultFactory();
if (sslSocketFactory != null) {
secureConn.setSSLSocketFactory(sslSocketFactory);
}

16
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
View File

@ -47,7 +47,7 @@
import static org.junit.Assert.assertNotEquals;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import org.junit.Test;
/**
@ -167,19 +167,19 @@ public void testGetAccountKeyWithNonExistingAccountName() throws Exception {
@Test
public void testSSLSocketFactoryConfiguration()
throws InvalidConfigurationValueException, IllegalAccessException, IOException {
assertEquals(SSLSocketFactoryEx.SSLChannelMode.Default, abfsConfiguration.getPreferredSSLFactoryOption());
assertNotEquals(SSLSocketFactoryEx.SSLChannelMode.Default_JSSE, abfsConfiguration.getPreferredSSLFactoryOption());
assertNotEquals(SSLSocketFactoryEx.SSLChannelMode.OpenSSL, abfsConfiguration.getPreferredSSLFactoryOption());
assertEquals(DelegatingSSLSocketFactory.SSLChannelMode.Default, abfsConfiguration.getPreferredSSLFactoryOption());
assertNotEquals(DelegatingSSLSocketFactory.SSLChannelMode.Default_JSSE, abfsConfiguration.getPreferredSSLFactoryOption());
assertNotEquals(DelegatingSSLSocketFactory.SSLChannelMode.OpenSSL, abfsConfiguration.getPreferredSSLFactoryOption());
Configuration configuration = new Configuration();
configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, SSLSocketFactoryEx.SSLChannelMode.Default_JSSE);
configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DelegatingSSLSocketFactory.SSLChannelMode.Default_JSSE);
AbfsConfiguration localAbfsConfiguration = new AbfsConfiguration(configuration, accountName);
assertEquals(SSLSocketFactoryEx.SSLChannelMode.Default_JSSE, localAbfsConfiguration.getPreferredSSLFactoryOption());
assertEquals(DelegatingSSLSocketFactory.SSLChannelMode.Default_JSSE, localAbfsConfiguration.getPreferredSSLFactoryOption());
configuration = new Configuration();
configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, SSLSocketFactoryEx.SSLChannelMode.OpenSSL);
configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DelegatingSSLSocketFactory.SSLChannelMode.OpenSSL);
localAbfsConfiguration = new AbfsConfiguration(configuration, accountName);
assertEquals(SSLSocketFactoryEx.SSLChannelMode.OpenSSL, localAbfsConfiguration.getPreferredSSLFactoryOption());
assertEquals(DelegatingSSLSocketFactory.SSLChannelMode.OpenSSL, localAbfsConfiguration.getPreferredSSLFactoryOption());
}
}

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
View File

@ -25,9 +25,9 @@
import org.junit.Test;
import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import org.apache.hadoop.util.VersionInfo;
/**
@ -46,7 +46,7 @@ private void validateUserAgent(String expectedPattern,
config, null, null);
String sslProviderName = null;
if (includeSSLProvider) {
sslProviderName = SSLSocketFactoryEx.getDefaultFactory().getProviderName();
sslProviderName = DelegatingSSLSocketFactory.getDefaultFactory().getProviderName();
}
String userAgent = client.initializeUserAgent(config, sslProviderName);
Pattern pattern = Pattern.compile(expectedPattern);
@ -86,7 +86,7 @@ public void verifyUserAgentWithSSLProvider() throws Exception {
final Configuration configuration = new Configuration();
configuration.set(ConfigurationKeys.FS_AZURE_USER_AGENT_PREFIX_KEY, "Partner Service");
configuration.set(ConfigurationKeys.FS_AZURE_SSL_CHANNEL_MODE_KEY,
SSLSocketFactoryEx.SSLChannelMode.Default_JSSE.name());
DelegatingSSLSocketFactory.SSLChannelMode.Default_JSSE.name());
AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, accountName);
validateUserAgent(expectedUserAgentPattern, new URL("https://azure.com"),
abfsConfiguration, true);