big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17337. S3A NetworkBinding has a runtime dependency on shaded httpclient. (#2599)

Browse Source 
Contributed by Steve Loughran.
This commit is contained in:
Steve Loughran 2021-02-03 14:29:56 +00:00 committed by GitHub
parent 66ecee333e
commit 26b9d480e8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 74 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.s3a.impl;
import javax.net.ssl.HostnameVerifier;
import java.io.IOException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.thirdparty.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
/**
* This interacts with the Shaded httpclient library used in the full
* AWS SDK. If the S3A client is used with the unshaded SDK, this
* class will not link.
*/
public class ConfigureShadedAWSSocketFactory implements
NetworkBinding.ConfigureAWSSocketFactory {
@Override
public void configureSocketFactory(final ClientConfiguration awsConf,
final DelegatingSSLSocketFactory.SSLChannelMode channelMode)
throws IOException {
DelegatingSSLSocketFactory.initializeDefaultFactory(channelMode);
awsConf.getApacheHttpClientConfig().setSslSocketFactory(
new SSLConnectionSocketFactory(
DelegatingSSLSocketFactory.getDefaultFactory(),
(HostnameVerifier) null));
}
}

52
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
View File

@ -19,14 +19,10 @@
package org.apache.hadoop.fs.s3a.impl;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.net.URISyntaxException;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLSocketFactory;
import com.amazonaws.ClientConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -43,20 +39,21 @@
/**
* Configures network settings when communicating with AWS services.
*/
public class NetworkBinding {
public final class NetworkBinding {
private static final Logger LOG =
LoggerFactory.getLogger(NetworkBinding.class);
private static final String AWS_SOCKET_FACTORY_CLASSNAME = "com.amazonaws" +
".thirdparty.apache.http.conn.ssl.SSLConnectionSocketFactory";
private static final String BINDING_CLASSNAME = "org.apache.hadoop.fs.s3a.impl.ConfigureShadedAWSSocketFactory";
private NetworkBinding() {
}
/**
* Configures the {@code SSLConnectionSocketFactory} used by the AWS SDK.
* A custom Socket Factory can be set using the method
* {@code setSslSocketFactory()}.
* If {@code SSLConnectionSocketFactory} cannot be found on the classpath, the value
* of {@link org.apache.hadoop.fs.s3a.Constants#SSL_CHANNEL_MODE} is ignored.
*
* Uses reflection to do this via {@link ConfigureShadedAWSSocketFactory}
* so as to avoid
* @param conf the {@link Configuration} used to get the client specified
* value of {@code SSL_CHANNEL_MODE}
* @param awsConf the {@code ClientConfiguration} to set the
@ -84,28 +81,33 @@ public static void bindSSLChannelMode(Configuration conf,
DelegatingSSLSocketFactory.initializeDefaultFactory(channelMode);
try {
// Look for AWS_SOCKET_FACTORY_CLASSNAME on the classpath and instantiate
// an instance using the DelegatingSSLSocketFactory as the
// SSLSocketFactory.
Class<?> sslConnectionSocketFactory = Class.forName(
AWS_SOCKET_FACTORY_CLASSNAME);
Constructor<?> factoryConstructor =
sslConnectionSocketFactory.getDeclaredConstructor(
SSLSocketFactory.class, HostnameVerifier.class);
awsConf.getApacheHttpClientConfig().setSslSocketFactory(
(com.amazonaws.thirdparty.apache.http.conn.ssl.
SSLConnectionSocketFactory) factoryConstructor
.newInstance(DelegatingSSLSocketFactory
.getDefaultFactory(),
(HostnameVerifier) null));
// use reflection to load in our own binding class.
// this is *probably* overkill, but it is how we can be fully confident
// that no attempt will be made to load/link to the AWS Shaded SDK except
// within this try/catch block
Class<? extends ConfigureAWSSocketFactory> clazz =
(Class<? extends ConfigureAWSSocketFactory>) Class.forName(BINDING_CLASSNAME);
clazz.getConstructor()
.newInstance()
.configureSocketFactory(awsConf, channelMode);
} catch (ClassNotFoundException | NoSuchMethodException |
IllegalAccessException | InstantiationException |
InvocationTargetException | LinkageError e) {
LOG.debug("Unable to create class {}, value of {} will be ignored",
AWS_SOCKET_FACTORY_CLASSNAME, SSL_CHANNEL_MODE, e);
BINDING_CLASSNAME, SSL_CHANNEL_MODE, e);
}
}
/**
* Interface used to bind to the socket factory, allows the code which
* works with the shaded AWS libraries to exist in their own class.
*/
interface ConfigureAWSSocketFactory {
void configureSocketFactory(ClientConfiguration awsConf,
DelegatingSSLSocketFactory.SSLChannelMode channelMode)
throws IOException;
}
/**
* Given an S3 bucket region as returned by a bucket location query,
* fix it into a form which can be used by other AWS commands.