diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 896ac4230a..aff03c20d5 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -346,6 +346,11 @@
org.wildfly.opensslwildfly-openssl
+ test
+
+
+ org.wildfly.openssl
+ wildfly-openssl-javaprovided
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java
index ad97a99c6d..c961364aa1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java
@@ -58,6 +58,10 @@
* SSL with no modification to the list of enabled ciphers.
*
*
+ *
+ * In order to load OpenSSL, applications must ensure the wildfly-openssl
+ * artifact is on the classpath. Currently, only ABFS and S3A provide
+ * wildfly-openssl as a runtime dependency.
*/
public final class DelegatingSSLSocketFactory extends SSLSocketFactory {
@@ -170,8 +174,14 @@ private void initializeSSLContext(SSLChannelMode preferredChannelMode)
OpenSSLProvider.register();
openSSLProviderRegistered = true;
}
+ java.util.logging.Logger logger = java.util.logging.Logger.getLogger(
+ SSL.class.getName());
+ logger.setLevel(Level.WARNING);
ctx = SSLContext.getInstance("openssl.TLS");
ctx.init(null, null, null);
+ // Strong reference needs to be kept to logger until initialization of
+ // SSLContext finished (see HADOOP-16174):
+ logger.setLevel(Level.INFO);
channelMode = SSLChannelMode.OpenSSL;
break;
case Default_JSSE:
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 9aadd74453..3e9beb9aad 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1978,11 +1978,16 @@
If secure connections to S3 are enabled, configures the SSL
implementation used to encrypt connections to S3. Supported values are:
- "default_jsse" and "default_jsse_with_gcm". "default_jsse" uses the Java
- Secure Socket Extension package (JSSE). However, when running on Java 8,
- the GCM cipher is removed from the list of enabled ciphers. This is due
- to performance issues with GCM in Java 8. "default_jsse_with_gcm" uses
- the JSSE with the default list of cipher suites.
+ "default_jsse", "default_jsse_with_gcm", "default", and "openssl".
+ "default_jsse" uses the Java Secure Socket Extension package (JSSE).
+ However, when running on Java 8, the GCM cipher is removed from the list
+ of enabled ciphers. This is due to performance issues with GCM in Java 8.
+ "default_jsse_with_gcm" uses the JSSE with the default list of cipher
+ suites. "default_jsse_with_gcm" is equivalent to the behavior prior to
+ this feature being introduced. "default" attempts to use OpenSSL rather
+ than the JSSE for SSL encryption, if OpenSSL libraries cannot be loaded,
+ it falls back to the "default_jsse" behavior. "openssl" attempts to use
+ OpenSSL as well, but fails if OpenSSL libraries cannot be loaded.
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 274bfd89f6..f5ffa76cd1 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -196,6 +196,7 @@
3.9.01.5.67.7.0
+ 1.0.7.Final
@@ -1370,7 +1371,12 @@
org.wildfly.opensslwildfly-openssl
- 1.0.7.Final
+ ${openssl-wildfly.version}
+
+
+ org.wildfly.openssl
+ wildfly-openssl-java
+ ${openssl-wildfly.version}
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index bd204b08a6..21c91dd5dd 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -430,6 +430,11 @@
assertj-coretest
+
+ org.wildfly.openssl
+ wildfly-openssl
+ runtime
+ junitjunit
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
index 94882f271f..7ff4451001 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
@@ -80,13 +80,6 @@ public static void bindSSLChannelMode(Configuration conf,
throw new IllegalArgumentException(channelModeString +
" is not a valid value for " + SSL_CHANNEL_MODE);
}
- if (channelMode == DelegatingSSLSocketFactory.SSLChannelMode.OpenSSL ||
- channelMode == DelegatingSSLSocketFactory.SSLChannelMode.Default) {
- throw new UnsupportedOperationException("S3A does not support " +
- "setting " + SSL_CHANNEL_MODE + " " +
- DelegatingSSLSocketFactory.SSLChannelMode.OpenSSL + " or " +
- DelegatingSSLSocketFactory.SSLChannelMode.Default);
- }
// Look for AWS_SOCKET_FACTORY_CLASSNAME on the classpath and instantiate
// an instance using the DelegatingSSLSocketFactory as the
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
index 6a9b3f7aba..5543263471 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
@@ -535,10 +535,41 @@ GCM cipher is only disabled on Java 8. GCM performance has been improved
in Java 9, so if `default_jsse` is specified and applications run on Java
9, they should see no difference compared to running with the vanilla JSSE.
-Other options for `fs.s3a.ssl.channel.mode` include `default_jsse_with_gcm`.
-This option includes GCM in the list of cipher suites on Java 8, so it is
-equivalent to running with the vanilla JSSE. The naming convention is setup
-in order to preserve backwards compatibility with HADOOP-15669.
+`fs.s3a.ssl.channel.mode` can be set to `default_jsse_with_gcm`. This option
+includes GCM in the list of cipher suites on Java 8, so it is equivalent to
+running with the vanilla JSSE.
+
+### OpenSSL Acceleration
+
+**Experimental Feature**
+
+As of HADOOP-16050 and HADOOP-16346, `fs.s3a.ssl.channel.mode` can be set to
+either `default` or `openssl` to enable native OpenSSL acceleration of HTTPS
+requests. OpenSSL implements the SSL and TLS protocols using native code. For
+users reading a large amount of data over HTTPS, OpenSSL can provide a
+significant performance benefit over the JSSE.
+
+S3A uses the
+[WildFly OpenSSL](https://github.com/wildfly-security/wildfly-openssl) library
+to bind OpenSSL to the Java JSSE APIs. This library allows S3A to
+transparently read data using OpenSSL. The wildfly-openssl library is a
+runtime dependency of S3A and contains native libraries for binding the Java
+JSSE to OpenSSL.
+
+WildFly OpenSSL must load OpenSSL itself. This can be done using the system
+property `org.wildfly.openssl.path`. For example,
+`HADOOP_OPTS="-Dorg.wildfly.openssl.path=
+${HADOOP_OPTS}"`. See WildFly OpenSSL documentation for more details.
+
+When `fs.s3a.ssl.channel.mode` is set to `default`, S3A will attempt to load
+the OpenSSL libraries using the WildFly library. If it is unsuccessful, it
+will fall back to the `default_jsse` behavior.
+
+When `fs.s3a.ssl.channel.mode` is set to `openssl`, S3A will attempt to load
+the OpenSSL libraries using WildFly. If it is unsuccessful, it will throw an
+exception and S3A initialization will fail.
+
+### `fs.s3a.ssl.channel.mode` Configuration
`fs.s3a.ssl.channel.mode` can be configured as follows:
@@ -549,11 +580,16 @@ in order to preserve backwards compatibility with HADOOP-15669.
If secure connections to S3 are enabled, configures the SSL
implementation used to encrypt connections to S3. Supported values are:
- "default_jsse" and "default_jsse_with_gcm". "default_jsse" uses the Java
- Secure Socket Extension package (JSSE). However, when running on Java 8,
- the GCM cipher is removed from the list of enabled ciphers. This is due
- to performance issues with GCM in Java 8. "default_jsse_with_gcm" uses
- the JSSE with the default list of cipher suites.
+ "default_jsse", "default_jsse_with_gcm", "default", and "openssl".
+ "default_jsse" uses the Java Secure Socket Extension package (JSSE).
+ However, when running on Java 8, the GCM cipher is removed from the list
+ of enabled ciphers. This is due to performance issues with GCM in Java 8.
+ "default_jsse_with_gcm" uses the JSSE with the default list of cipher
+ suites. "default_jsse_with_gcm" is equivalent to the behavior prior to
+ this feature being introduced. "default" attempts to use OpenSSL rather
+ than the JSSE for SSL encryption, if OpenSSL libraries cannot be loaded,
+ it falls back to the "default_jsse" behavior. "openssl" attempts to use
+ OpenSSL as well, but fails if OpenSSL libraries cannot be loaded.
```
@@ -564,6 +600,11 @@ Supported values for `fs.s3a.ssl.channel.mode`:
|-------------------------------|-------------|
| default_jsse | Uses Java JSSE without GCM on Java 8 |
| default_jsse_with_gcm | Uses Java JSSE |
+| default | Uses OpenSSL, falls back to default_jsse if OpenSSL cannot be loaded |
+| openssl | Uses OpenSSL, fails if OpenSSL cannot be loaded |
+
+The naming convention is setup in order to preserve backwards compatibility
+with HADOOP-15669.
Other options may be added to `fs.s3a.ssl.channel.mode` in the future as
-further SSL optimizations are made.
\ No newline at end of file
+further SSL optimizations are made.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
index 9332621d11..fca1004ae9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
@@ -24,6 +24,7 @@
import java.util.Arrays;
import java.util.Collection;
+import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -41,6 +42,7 @@
import org.apache.hadoop.fs.s3a.S3AInputPolicy;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
+import org.apache.hadoop.util.NativeCodeLoader;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADVISE;
@@ -55,6 +57,9 @@
SSLChannelMode.Default_JSSE;
import static org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory.
SSLChannelMode.Default_JSSE_with_GCM;
+import static org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory.
+ SSLChannelMode.OpenSSL;
+import static org.junit.Assume.assumeTrue;
/**
@@ -84,7 +89,7 @@ public class ITestS3AContractSeek extends AbstractContractSeekTest {
public static Collection