From d6a4f39bd5f192e5e3377630887a6998d4d400c2 Mon Sep 17 00:00:00 2001 From: Thomas Marquardt Date: Thu, 16 Aug 2018 16:19:13 +0000 Subject: [PATCH] HADOOP-15669. ABFS: Improve HTTPS Performance. Contributed by Vishwajeet Dusane. --- hadoop-project/pom.xml | 7 + hadoop-tools/hadoop-azure/pom.xml | 7 +- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 8 + .../azurebfs/constants/ConfigurationKeys.java | 1 + .../constants/FileSystemConfigurations.java | 4 + .../fs/azurebfs/services/AbfsClient.java | 43 +++- .../azurebfs/services/AbfsHttpOperation.java | 11 + .../fs/azurebfs/utils/SSLSocketFactoryEx.java | 240 ++++++++++++++++++ ...TestAbfsConfigurationFieldsValidation.java | 42 ++- .../fs/azurebfs/services/TestAbfsClient.java | 40 ++- 10 files changed, 375 insertions(+), 28 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/SSLSocketFactoryEx.java diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 184cb3d1ac..a13283a0ac 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1349,6 +1349,13 @@ 7.0.0 + + + org.wildfly.openssl + wildfly-openssl + 1.0.4.Final + + org.threadly threadly diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 7d0406c6cf..7152f6383a 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -197,13 +197,18 @@ jackson-mapper-asl compile + org.codehaus.jackson jackson-core-asl compile - + + org.wildfly.openssl + wildfly-openssl + compile + diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 1fb5df9aa3..e647ae8f25 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -44,6 +44,10 @@ import org.apache.hadoop.fs.azurebfs.diagnostics.StringConfigurationBasicValidator; import org.apache.hadoop.fs.azurebfs.services.KeyProvider; import org.apache.hadoop.fs.azurebfs.services.SimpleKeyProvider; +import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SSL_CHANNEL_MODE_KEY; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_SSL_CHANNEL_MODE; /** * Configuration for Azure Blob FileSystem. @@ -270,6 +274,10 @@ public String getCustomUserAgentPrefix() { return this.userAgentId; } + public SSLSocketFactoryEx.SSLChannelMode getPreferredSSLFactoryOption() { + return configuration.getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE); + } + void validateStorageAccountKeys() throws InvalidConfigurationValueException { Base64StringConfigurationBasicValidator validator = new Base64StringConfigurationBasicValidator( ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, "", true); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 9c805a2a7c..16ddd900ed 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -55,6 +55,7 @@ public final class ConfigurationKeys { public static final String FS_AZURE_READ_AHEAD_QUEUE_DEPTH = "fs.azure.readaheadqueue.depth"; public static final String FS_AZURE_ENABLE_FLUSH = "fs.azure.enable.flush"; public static final String FS_AZURE_USER_AGENT_PREFIX_KEY = "fs.azure.user.agent.prefix"; + public static final String FS_AZURE_SSL_CHANNEL_MODE_KEY = "fs.azure.ssl.channel.mode"; public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider."; public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 1655d04049..5b92dddf73 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx; /** * Responsible to keep all the Azure Blob File System related configurations. @@ -57,5 +58,8 @@ public final class FileSystemConfigurations { public static final int DEFAULT_READ_AHEAD_QUEUE_DEPTH = -1; public static final boolean DEFAULT_ENABLE_FLUSH = true; + public static final SSLSocketFactoryEx.SSLChannelMode DEFAULT_FS_AZURE_SSL_CHANNEL_MODE + = SSLSocketFactoryEx.SSLChannelMode.Default; + private FileSystemConfigurations() {} } \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 60369be9bc..e003ffd31d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.azurebfs.services; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; @@ -27,6 +28,7 @@ import java.util.Locale; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +37,7 @@ import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.HTTPS_SCHEME; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.*; import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.*; @@ -60,7 +63,19 @@ public AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredent this.filesystem = baseUrlString.substring(baseUrlString.lastIndexOf(FORWARD_SLASH) + 1); this.abfsConfiguration = abfsConfiguration; this.retryPolicy = exponentialRetryPolicy; - this.userAgent = initializeUserAgent(abfsConfiguration); + + String sslProviderName = null; + + if (this.baseUrl.toString().startsWith(HTTPS_SCHEME)) { + try { + SSLSocketFactoryEx.initializeDefaultFactory(this.abfsConfiguration.getPreferredSSLFactoryOption()); + sslProviderName = SSLSocketFactoryEx.getDefaultFactory().getProviderName(); + } catch (IOException e) { + // Suppress exception. Failure to init SSLSocketFactoryEx would have only performance impact. + } + } + + this.userAgent = initializeUserAgent(abfsConfiguration, sslProviderName); } public String getFileSystem() { @@ -395,16 +410,26 @@ public static String urlEncode(final String value) throws AzureBlobFileSystemExc } @VisibleForTesting - String initializeUserAgent(final AbfsConfiguration abfsConfiguration) { - final String userAgentComment = String.format(Locale.ROOT, - "(JavaJRE %s; %s %s)", - System.getProperty(JAVA_VERSION), - System.getProperty(OS_NAME) - .replaceAll(SINGLE_WHITE_SPACE, EMPTY_STRING), - System.getProperty(OS_VERSION)); + String initializeUserAgent(final AbfsConfiguration abfsConfiguration, + final String sslProviderName) { + StringBuilder sb = new StringBuilder(); + sb.append("(JavaJRE "); + sb.append(System.getProperty(JAVA_VERSION)); + sb.append("; "); + sb.append( + System.getProperty(OS_NAME).replaceAll(SINGLE_WHITE_SPACE, EMPTY_STRING)); + sb.append(" "); + sb.append(System.getProperty(OS_VERSION)); + if (sslProviderName != null && !sslProviderName.isEmpty()) { + sb.append("; "); + sb.append(sslProviderName); + } + sb.append(")"); + final String userAgentComment = sb.toString(); String customUserAgentId = abfsConfiguration.getCustomUserAgentPrefix(); if (customUserAgentId != null && !customUserAgentId.isEmpty()) { - return String.format(Locale.ROOT, CLIENT_VERSION + " %s %s", userAgentComment, customUserAgentId); + return String.format(Locale.ROOT, CLIENT_VERSION + " %s %s", + userAgentComment, customUserAgentId); } return String.format(CLIENT_VERSION + " %s", userAgentComment); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 2bfcff2500..f493298f2c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -26,6 +26,10 @@ import java.util.List; import java.util.UUID; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLSocketFactory; + +import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.JsonToken; @@ -174,6 +178,13 @@ public AbfsHttpOperation(final URL url, final String method, final List preferredSuits = new ArrayList<>(); + + // Remove GCM mode based ciphers from the supported list. + for (int i = 0; i < defaultCiphers.length; i++) { + if (defaultCiphers[i].contains("_GCM_")) { + LOG.debug("Removed Cipher - " + defaultCiphers[i]); + } else { + preferredSuits.add(defaultCiphers[i]); + } + } + + ciphers = preferredSuits.toArray(new String[0]); + return ciphers; + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java index fb667ddc8a..556dcdb042 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.Base64StringConfigurationValidatorAnnotation; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SSL_CHANNEL_MODE_KEY; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_MAX_RETRY_ATTEMPTS; @@ -41,7 +42,10 @@ import org.apache.commons.codec.binary.Base64; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; +import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx; import org.junit.Test; /** @@ -50,11 +54,11 @@ public class TestAbfsConfigurationFieldsValidation { private AbfsConfiguration abfsConfiguration; - private static final String INT_KEY= "intKey"; - private static final String LONG_KEY= "longKey"; - private static final String STRING_KEY= "stringKey"; - private static final String BASE64_KEY= "base64Key"; - private static final String BOOLEAN_KEY= "booleanKey"; + private static final String INT_KEY = "intKey"; + private static final String LONG_KEY = "longKey"; + private static final String STRING_KEY = "stringKey"; + private static final String BASE64_KEY = "base64Key"; + private static final String BOOLEAN_KEY = "booleanKey"; private static final int DEFAULT_INT = 4194304; private static final int DEFAULT_LONG = 4194304; @@ -77,15 +81,15 @@ public class TestAbfsConfigurationFieldsValidation { private int longField; @StringConfigurationValidatorAnnotation(ConfigurationKey = STRING_KEY, - DefaultValue = "default") + DefaultValue = "default") private String stringField; @Base64StringConfigurationValidatorAnnotation(ConfigurationKey = BASE64_KEY, - DefaultValue = "base64") + DefaultValue = "base64") private String base64Field; @BooleanConfigurationValidatorAnnotation(ConfigurationKey = BOOLEAN_KEY, - DefaultValue = false) + DefaultValue = false) private boolean boolField; public TestAbfsConfigurationFieldsValidation() throws Exception { @@ -142,8 +146,26 @@ public void testGetAccountKey() throws Exception { assertEquals(this.encodedAccountKey, accountKey); } - @Test (expected = ConfigurationPropertyNotFoundException.class) + @Test(expected = ConfigurationPropertyNotFoundException.class) public void testGetAccountKeyWithNonExistingAccountName() throws Exception { abfsConfiguration.getStorageAccountKey("bogusAccountName"); } -} + + @Test + public void testSSLSocketFactoryConfiguration() throws InvalidConfigurationValueException, IllegalAccessException { + assertEquals(SSLSocketFactoryEx.SSLChannelMode.Default, abfsConfiguration.getPreferredSSLFactoryOption()); + assertNotEquals(SSLSocketFactoryEx.SSLChannelMode.Default_JSSE, abfsConfiguration.getPreferredSSLFactoryOption()); + assertNotEquals(SSLSocketFactoryEx.SSLChannelMode.OpenSSL, abfsConfiguration.getPreferredSSLFactoryOption()); + + Configuration configuration = new Configuration(); + configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, SSLSocketFactoryEx.SSLChannelMode.Default_JSSE); + AbfsConfiguration localAbfsConfiguration = new AbfsConfiguration(configuration); + assertEquals(SSLSocketFactoryEx.SSLChannelMode.Default_JSSE, localAbfsConfiguration.getPreferredSSLFactoryOption()); + + configuration = new Configuration(); + configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, SSLSocketFactoryEx.SSLChannelMode.OpenSSL); + localAbfsConfiguration = new AbfsConfiguration(configuration); + assertEquals(SSLSocketFactoryEx.SSLChannelMode.OpenSSL, localAbfsConfiguration.getPreferredSSLFactoryOption()); + } + +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index 0b335a53e0..7bb27fc451 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -22,6 +22,7 @@ import java.util.regex.Pattern; import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx; import org.junit.Assert; import org.junit.Test; @@ -34,16 +35,29 @@ */ public final class TestAbfsClient { + private void validateUserAgent(String expectedPattern, + URL baseUrl, + AbfsConfiguration config, + boolean includeSSLProvider) { + AbfsClient client = new AbfsClient(baseUrl, null, + config, null); + String sslProviderName = null; + if (includeSSLProvider) { + sslProviderName = SSLSocketFactoryEx.getDefaultFactory().getProviderName(); + } + String userAgent = client.initializeUserAgent(config, sslProviderName); + Pattern pattern = Pattern.compile(expectedPattern); + Assert.assertTrue(pattern.matcher(userAgent).matches()); + } + @Test public void verifyUnknownUserAgent() throws Exception { String expectedUserAgentPattern = "Azure Blob FS\\/1.0 \\(JavaJRE ([^\\)]+)\\)"; final Configuration configuration = new Configuration(); configuration.unset(ConfigurationKeys.FS_AZURE_USER_AGENT_PREFIX_KEY); AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration); - AbfsClient abfsClient = new AbfsClient(new URL("http://azure.com"), null, abfsConfiguration, null); - String userAgent = abfsClient.initializeUserAgent(abfsConfiguration); - Pattern pattern = Pattern.compile(expectedUserAgentPattern); - Assert.assertTrue(pattern.matcher(userAgent).matches()); + validateUserAgent(expectedUserAgentPattern, new URL("http://azure.com"), + abfsConfiguration, false); } @Test @@ -52,9 +66,19 @@ public void verifyUserAgent() throws Exception { final Configuration configuration = new Configuration(); configuration.set(ConfigurationKeys.FS_AZURE_USER_AGENT_PREFIX_KEY, "Partner Service"); AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration); - AbfsClient abfsClient = new AbfsClient(new URL("http://azure.com"), null, abfsConfiguration, null); - String userAgent = abfsClient.initializeUserAgent(abfsConfiguration); - Pattern pattern = Pattern.compile(expectedUserAgentPattern); - Assert.assertTrue(pattern.matcher(userAgent).matches()); + validateUserAgent(expectedUserAgentPattern, new URL("http://azure.com"), + abfsConfiguration, false); + } + + @Test + public void verifyUserAgentWithSSLProvider() throws Exception { + String expectedUserAgentPattern = "Azure Blob FS\\/1.0 \\(JavaJRE ([^\\)]+) SunJSSE-1.8\\) Partner Service"; + final Configuration configuration = new Configuration(); + configuration.set(ConfigurationKeys.FS_AZURE_USER_AGENT_PREFIX_KEY, "Partner Service"); + configuration.set(ConfigurationKeys.FS_AZURE_SSL_CHANNEL_MODE_KEY, + SSLSocketFactoryEx.SSLChannelMode.Default_JSSE.name()); + AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration); + validateUserAgent(expectedUserAgentPattern, new URL("https://azure.com"), + abfsConfiguration, true); } } \ No newline at end of file