From 62579b69a0a294ba1ea14cf76c650b640f89f331 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Thu, 27 Apr 2017 16:57:08 +0900 Subject: [PATCH] HADOOP-11614. Remove httpclient dependency from hadoop-openstack. Contributed by Akira Ajisaka, Brahma Reddy Battula, and Steve Loughran. --- hadoop-tools/hadoop-openstack/pom.xml | 6 +- .../SwiftAuthenticationFailedException.java | 6 +- .../exceptions/SwiftBadRequestException.java | 6 +- .../SwiftInvalidResponseException.java | 9 +- .../SwiftThrottledRequestException.java | 6 +- .../{CopyMethod.java => CopyRequest.java} | 12 +- .../http/HttpInputStreamWithRelease.java | 25 +- .../hadoop/fs/swift/http/SwiftRestClient.java | 791 +++++++++--------- .../snative/SwiftNativeFileSystemStore.java | 11 +- .../fs/swift/util/HttpResponseUtils.java | 121 +++ ...TestSwiftFileSystemPartitionedUploads.java | 2 +- .../fs/swift/http/TestSwiftRestClient.java | 2 +- .../src/test/resources/log4j.properties | 3 - 13 files changed, 570 insertions(+), 430 deletions(-) rename hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/{CopyMethod.java => CopyRequest.java} (85%) create mode 100644 hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/HttpResponseUtils.java diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml index add9fe9364..0e0defb86a 100644 --- a/hadoop-tools/hadoop-openstack/pom.xml +++ b/hadoop-tools/hadoop-openstack/pom.xml @@ -124,10 +124,8 @@ - commons-httpclient - commons-httpclient - 3.1 - compile + org.apache.httpcomponents + httpcore commons-logging diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftAuthenticationFailedException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftAuthenticationFailedException.java index e54c0fb1df..fdb9a3973a 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftAuthenticationFailedException.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftAuthenticationFailedException.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.swift.exceptions; -import org.apache.commons.httpclient.HttpMethod; +import org.apache.http.HttpResponse; import java.net.URI; @@ -37,8 +37,8 @@ public SwiftAuthenticationFailedException(String message, public SwiftAuthenticationFailedException(String message, String operation, URI uri, - HttpMethod method) { - super(message, operation, uri, method); + HttpResponse resp) { + super(message, operation, uri, resp); } @Override diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftBadRequestException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftBadRequestException.java index 65910e9203..f5b2abde0a 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftBadRequestException.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftBadRequestException.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.swift.exceptions; -import org.apache.commons.httpclient.HttpMethod; +import org.apache.http.HttpResponse; import java.net.URI; @@ -31,8 +31,8 @@ public class SwiftBadRequestException extends SwiftInvalidResponseException { public SwiftBadRequestException(String message, String operation, URI uri, - HttpMethod method) { - super(message, operation, uri, method); + HttpResponse resp) { + super(message, operation, uri, resp); } public SwiftBadRequestException(String message, diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInvalidResponseException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInvalidResponseException.java index 021518f1ad..e90e57519b 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInvalidResponseException.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInvalidResponseException.java @@ -18,7 +18,8 @@ package org.apache.hadoop.fs.swift.exceptions; -import org.apache.commons.httpclient.HttpMethod; +import org.apache.hadoop.fs.swift.util.HttpResponseUtils; +import org.apache.http.HttpResponse; import java.io.IOException; import java.net.URI; @@ -48,14 +49,14 @@ public SwiftInvalidResponseException(String message, public SwiftInvalidResponseException(String message, String operation, URI uri, - HttpMethod method) { + HttpResponse resp) { super(message); - this.statusCode = method.getStatusCode(); + this.statusCode = resp.getStatusLine().getStatusCode(); this.operation = operation; this.uri = uri; String bodyAsString; try { - bodyAsString = method.getResponseBodyAsString(); + bodyAsString = HttpResponseUtils.getResponseBodyAsString(resp); if (bodyAsString == null) { bodyAsString = ""; } diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java index 0553b45df9..1e7ca67d1b 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.swift.exceptions; -import org.apache.commons.httpclient.HttpMethod; +import org.apache.http.HttpResponse; import java.net.URI; @@ -31,7 +31,7 @@ public class SwiftThrottledRequestException extends public SwiftThrottledRequestException(String message, String operation, URI uri, - HttpMethod method) { - super(message, operation, uri, method); + HttpResponse resp) { + super(message, operation, uri, resp); } } diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyMethod.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyRequest.java similarity index 85% rename from hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyMethod.java rename to hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyRequest.java index d314aff08b..c25a630cc2 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyMethod.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyRequest.java @@ -18,24 +18,24 @@ package org.apache.hadoop.fs.swift.http; -import org.apache.commons.httpclient.methods.EntityEnclosingMethod; +import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; /** * Implementation for SwiftRestClient to make copy requests. * COPY is a method that came with WebDAV (RFC2518), and is not something that * can be handled by all proxies en-route to a filesystem. */ -class CopyMethod extends EntityEnclosingMethod { +class CopyRequest extends HttpEntityEnclosingRequestBase { - public CopyMethod(String uri) { - super(uri); + CopyRequest() { + super(); } /** * @return http method name */ @Override - public String getName() { + public String getMethod() { return "COPY"; } -} +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java index 627792cb58..aa52f8317c 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java @@ -18,11 +18,12 @@ package org.apache.hadoop.fs.swift.http; -import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException; import org.apache.hadoop.fs.swift.util.SwiftUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpRequestBase; import java.io.ByteArrayInputStream; import java.io.EOFException; @@ -46,7 +47,8 @@ public class HttpInputStreamWithRelease extends InputStream { private static final Log LOG = LogFactory.getLog(HttpInputStreamWithRelease.class); private final URI uri; - private HttpMethod method; + private HttpRequestBase req; + private HttpResponse resp; //flag to say the stream is released -volatile so that read operations //pick it up even while unsynchronized. private volatile boolean released; @@ -64,16 +66,17 @@ public class HttpInputStreamWithRelease extends InputStream { */ private String reasonClosed = "unopened"; - public HttpInputStreamWithRelease(URI uri, HttpMethod method) throws - IOException { + public HttpInputStreamWithRelease(URI uri, HttpRequestBase req, + HttpResponse resp) throws IOException { this.uri = uri; - this.method = method; + this.req = req; + this.resp = resp; constructionStack = LOG.isDebugEnabled() ? new Exception("stack") : null; - if (method == null) { - throw new IllegalArgumentException("Null 'method' parameter "); + if (req == null) { + throw new IllegalArgumentException("Null 'request' parameter "); } try { - inStream = method.getResponseBodyAsStream(); + inStream = resp.getEntity().getContent(); } catch (IOException e) { inStream = new ByteArrayInputStream(new byte[]{}); throw releaseAndRethrow("getResponseBodyAsStream() in constructor -" + e, e); @@ -100,11 +103,11 @@ private synchronized boolean release(String reason, Exception ex) throws if (LOG.isDebugEnabled()) { LOG.debug("Releasing connection to " + uri + ": " + reason, ex); } - if (method != null) { + if (req != null) { if (!dataConsumed) { - method.abort(); + req.abort(); } - method.releaseConnection(); + req.releaseConnection(); } if (inStream != null) { //this guard may seem un-needed, but a stack trace seen diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java index b8654cee80..ee6cc2fc3e 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java @@ -18,22 +18,28 @@ package org.apache.hadoop.fs.swift.http; -import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler; -import org.apache.commons.httpclient.Header; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpHost; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.HttpMethodBase; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.httpclient.methods.DeleteMethod; -import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.methods.HeadMethod; -import org.apache.commons.httpclient.methods.InputStreamRequestEntity; -import org.apache.commons.httpclient.methods.PostMethod; -import org.apache.commons.httpclient.methods.PutMethod; -import org.apache.commons.httpclient.methods.StringRequestEntity; -import org.apache.commons.httpclient.params.HttpConnectionParams; -import org.apache.commons.httpclient.params.HttpMethodParams; +import org.apache.hadoop.fs.swift.util.HttpResponseUtils; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.config.SocketConfig; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicHeader; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -63,7 +69,6 @@ import org.apache.hadoop.fs.swift.util.JSONUtil; import org.apache.hadoop.fs.swift.util.SwiftObjectPath; import org.apache.hadoop.fs.swift.util.SwiftUtils; -import org.apache.http.conn.params.ConnRoutePNames; import java.io.EOFException; import java.io.FileNotFoundException; @@ -76,11 +81,11 @@ import java.util.List; import java.util.Properties; -import static org.apache.commons.httpclient.HttpStatus.*; +import static org.apache.http.HttpStatus.*; import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.*; /** - * This implements the client-side of the Swift REST API + * This implements the client-side of the Swift REST API. * * The core actions put, get and query data in the Swift object store, * after authenticating the client. @@ -102,10 +107,10 @@ public final class SwiftRestClient { * consistency can be made. */ public static final Header NEWEST = - new Header(SwiftProtocolConstants.X_NEWEST, "true"); + new BasicHeader(SwiftProtocolConstants.X_NEWEST, "true"); /** - * the authentication endpoint as supplied in the configuration + * the authentication endpoint as supplied in the configuration. */ private final URI authUri; @@ -116,52 +121,52 @@ public final class SwiftRestClient { private final String region; /** - * tenant name + * tenant name. */ private final String tenant; /** - * username name + * username name. */ private final String username; /** - * user password + * user password. */ private final String password; /** - * user api key + * user api key. */ private final String apiKey; /** - * The authentication request used to authenticate with Swift + * The authentication request used to authenticate with Swift. */ private final AuthenticationRequest authRequest; /** * This auth request is similar to @see authRequest, * with one difference: it has another json representation when - * authRequest one is not applicable + * authRequest one is not applicable. */ private AuthenticationRequest keystoneAuthRequest; private boolean useKeystoneAuthentication = false; - /** - * The container this client is working with + /** + * The container this client is working with. */ private final String container; private final String serviceDescription; /** - * Access token (Secret) + * Access token (Secret). */ private AccessToken token; /** - * Endpoint for swift operations, obtained after authentication + * Endpoint for swift operations, obtained after authentication. */ private URI endpointURI; @@ -173,7 +178,7 @@ public final class SwiftRestClient { private URI objectLocationURI; /** - * The name of the service provider + * The name of the service provider. */ private final String serviceProvider; @@ -183,32 +188,32 @@ public final class SwiftRestClient { private final boolean usePublicURL; /** - * Number of times to retry a connection + * Number of times to retry a connection. */ private final int retryCount; /** - * How long (in milliseconds) should a connection be attempted + * How long (in milliseconds) should a connection be attempted. */ private final int connectTimeout; /** - * How long (in milliseconds) should a connection be attempted + * How long (in milliseconds) should a connection be attempted. */ private final int socketTimeout; /** - * How long (in milliseconds) between bulk operations + * How long (in milliseconds) between bulk operations. */ private final int throttleDelay; /** - * the name of a proxy host (can be null, in which case there is no proxy) + * the name of a proxy host (can be null, in which case there is no proxy). */ private String proxyHost; /** - * The port of a proxy. This is ignored if {@link #proxyHost} is null + * The port of a proxy. This is ignored if {@link #proxyHost} is null. */ private int proxyPort; @@ -236,7 +241,7 @@ private synchronized URI getEndpointURI() { } /** - * token for Swift communication + * token for Swift communication. */ private synchronized AccessToken getToken() { return token; @@ -268,39 +273,41 @@ private void setAuthDetails(URI endpoint, /** - * Base class for all Swift REST operations + * Base class for all Swift REST operations. * - * @param method + * @param request * @param result */ - private static abstract class HttpMethodProcessor { - public final M createMethod(String uri) throws IOException { - final M method = doCreateMethod(uri); - setup(method); - return method; + private static abstract class HttpRequestProcessor + { + public final M createRequest(String uri) throws IOException { + final M req = doCreateRequest(uri); + setup(req); + return req; } /** - * Override it to return some result after method is executed. + * Override it to return some result after request is executed. */ - public abstract R extractResult(M method) throws IOException; + public abstract R extractResult(M req, HttpResponse resp) + throws IOException; /** - * Factory method to create a REST method against the given URI + * Factory method to create a REST method against the given URI. * * @param uri target * @return method to invoke */ - protected abstract M doCreateMethod(String uri); + protected abstract M doCreateRequest(String uri) throws IOException; /** - * Override port to set up the method before it is executed. + * Override port to set up the request before it is executed. */ - protected void setup(M method) throws IOException { + protected void setup(M req) throws IOException { } /** - * Override point: what are the status codes that this operation supports + * Override point: what are the status codes that this operation supports? * * @return an array with the permitted status code(s) */ @@ -315,10 +322,19 @@ protected int[] getAllowedStatusCodes() { } } - private static abstract class GetMethodProcessor extends HttpMethodProcessor { + private static abstract class GetRequestProcessor + extends HttpRequestProcessor { @Override - protected final GetMethod doCreateMethod(String uri) { - return new GetMethod(uri); + protected final HttpGet doCreateRequest(String uri) { + return new HttpGet(uri); + } + } + + private static abstract class PostRequestProcessor + extends HttpRequestProcessor { + @Override + protected final HttpPost doCreateRequest(String uri) { + return new HttpPost(uri); } } @@ -327,34 +343,33 @@ protected final GetMethod doCreateMethod(String uri) { * message handlers can react to auth failures differently from everything * else. */ - private static class AuthPostMethod extends PostMethod { - - - private AuthPostMethod(String uri) { + private static final class AuthPostRequest extends HttpPost { + private AuthPostRequest(String uri) { super(uri); } } /** - * Generate an auth message + * Generate an auth message. * @param response */ - private static abstract class AuthMethodProcessor extends - HttpMethodProcessor { + private static abstract class AuthRequestProcessor + extends HttpRequestProcessor { @Override - protected final AuthPostMethod doCreateMethod(String uri) { - return new AuthPostMethod(uri); + protected final AuthPostRequest doCreateRequest(String uri) { + return new AuthPostRequest(uri); } } - private static abstract class PutMethodProcessor extends HttpMethodProcessor { + private static abstract class PutRequestProcessor + extends HttpRequestProcessor { @Override - protected final PutMethod doCreateMethod(String uri) { - return new PutMethod(uri); + protected final HttpPut doCreateRequest(String uri) { + return new HttpPut(uri); } /** - * Override point: what are the status codes that this operation supports + * Override point: what are the status codes that this operation supports? * * @return the list of status codes to accept */ @@ -370,18 +385,26 @@ protected int[] getAllowedStatusCodes() { } /** - * Create operation + * Create operation. * - * @param + * @param result type */ - private static abstract class CopyMethodProcessor extends HttpMethodProcessor { + private static abstract class CopyRequestProcessor + extends HttpRequestProcessor { @Override - protected final CopyMethod doCreateMethod(String uri) { - return new CopyMethod(uri); + protected final CopyRequest doCreateRequest(String uri) + throws SwiftException { + CopyRequest copy = new CopyRequest(); + try { + copy.setURI(new URI(uri)); + } catch (URISyntaxException e) { + throw new SwiftException("Failed to create URI from: " + uri); + } + return copy; } /** - * The only allowed status code is 201:created + * The only allowed status code is 201:created. * @return an array with the permitted status code(s) */ @Override @@ -393,14 +416,15 @@ protected int[] getAllowedStatusCodes() { } /** - * Delete operation + * Delete operation. * * @param */ - private static abstract class DeleteMethodProcessor extends HttpMethodProcessor { + private static abstract class DeleteRequestProcessor + extends HttpRequestProcessor { @Override - protected final DeleteMethod doCreateMethod(String uri) { - return new DeleteMethod(uri); + protected final HttpDelete doCreateRequest(String uri) { + return new HttpDelete(uri); } @Override @@ -414,10 +438,11 @@ protected int[] getAllowedStatusCodes() { } } - private static abstract class HeadMethodProcessor extends HttpMethodProcessor { + private static abstract class HeadRequestProcessor + extends HttpRequestProcessor { @Override - protected final HeadMethod doCreateMethod(String uri) { - return new HeadMethod(uri); + protected final HttpHead doCreateRequest(String uri) { + return new HttpHead(uri); } } @@ -555,7 +580,7 @@ private SwiftRestClient(URI filesystemURI, } /** - * Get a mandatory configuration option + * Get a mandatory configuration option. * * @param props property set * @param key key @@ -606,12 +631,12 @@ public HttpBodyContent getData(SwiftObjectPath path, } return getData(path, - new Header(HEADER_RANGE, range), + new BasicHeader(HEADER_RANGE, range), SwiftRestClient.NEWEST); } /** - * Returns object length + * Returns object length. * * @param uri file URI * @return object length @@ -620,22 +645,23 @@ public HttpBodyContent getData(SwiftObjectPath path, */ public long getContentLength(URI uri) throws IOException { preRemoteCommand("getContentLength"); - return perform("getContentLength", uri, new HeadMethodProcessor() { + return perform("getContentLength", uri, new HeadRequestProcessor() { @Override - public Long extractResult(HeadMethod method) throws IOException { - return method.getResponseContentLength(); + public Long extractResult(HttpHead req, HttpResponse resp) + throws IOException { + return HttpResponseUtils.getContentLength(resp); } @Override - protected void setup(HeadMethod method) throws IOException { - super.setup(method); - method.addRequestHeader(NEWEST); + protected void setup(HttpHead req) throws IOException { + super.setup(req); + req.addHeader(NEWEST); } }); } /** - * Get the length of the remote object + * Get the length of the remote object. * @param path object to probe * @return the content length * @throws IOException on any failure @@ -664,7 +690,7 @@ public HttpBodyContent getData(SwiftObjectPath path, } /** - * Returns object location as byte[] + * Returns object location as byte[]. * * @param path path to file * @param requestHeaders http headers @@ -680,39 +706,42 @@ public byte[] getObjectLocation(SwiftObjectPath path, preRemoteCommand("getObjectLocation"); try { return perform("getObjectLocation", pathToObjectLocation(path), - new GetMethodProcessor() { - @Override - protected int[] getAllowedStatusCodes() { - return new int[]{ - SC_OK, - SC_FORBIDDEN, - SC_NO_CONTENT - }; - } + new GetRequestProcessor() { + @Override + protected int[] getAllowedStatusCodes() { + return new int[]{ + SC_OK, + SC_FORBIDDEN, + SC_NO_CONTENT + }; + } - @Override - public byte[] extractResult(GetMethod method) throws - IOException { + @Override + public byte[] extractResult(HttpGet req, HttpResponse resp) throws + IOException { - //TODO: remove SC_NO_CONTENT if it depends on Swift versions - if (method.getStatusCode() == SC_NOT_FOUND - || method.getStatusCode() == SC_FORBIDDEN || - method.getStatusCode() == SC_NO_CONTENT || - method.getResponseBodyAsStream() == null) { - return null; - } - final InputStream responseBodyAsStream = method.getResponseBodyAsStream(); - final byte[] locationData = new byte[1024]; + //TODO: remove SC_NO_CONTENT if it depends on Swift versions + int statusCode = resp.getStatusLine().getStatusCode(); + if (statusCode == SC_NOT_FOUND + || statusCode == SC_FORBIDDEN + || statusCode == SC_NO_CONTENT + || resp.getEntity().getContent() == null) { + return null; + } + final InputStream responseBodyAsStream = + resp.getEntity().getContent(); + final byte[] locationData = new byte[1024]; - return responseBodyAsStream.read(locationData) > 0 ? locationData : null; - } + return responseBodyAsStream.read(locationData) > 0 ? + locationData : null; + } - @Override - protected void setup(GetMethod method) - throws SwiftInternalStateException { - setHeaders(method, requestHeaders); - } - }); + @Override + protected void setup(HttpGet req) + throws SwiftInternalStateException { + setHeaders(req, requestHeaders); + } + }); } catch (IOException e) { LOG.warn("Failed to get the location of " + path + ": " + e, e); return null; @@ -720,7 +749,7 @@ protected void setup(GetMethod method) } /** - * Create the URI needed to query the location of an object + * Create the URI needed to query the location of an object. * @param path object path to retrieve information about * @return the URI for the location operation * @throws SwiftException if the URI could not be constructed @@ -743,7 +772,7 @@ private URI pathToObjectLocation(SwiftObjectPath path) throws SwiftException { } /** - * Find objects under a prefix + * Find objects under a prefix. * * @param path path prefix * @param requestHeaders optional request headers @@ -753,7 +782,7 @@ private URI pathToObjectLocation(SwiftObjectPath path) throws SwiftException { * the directory is empty */ public byte[] findObjectsByPrefix(SwiftObjectPath path, - final Header... requestHeaders) throws IOException { + final Header... requestHeaders) throws IOException { preRemoteCommand("findObjectsByPrefix"); URI uri; String dataLocationURI = getEndpointURI().toString(); @@ -773,34 +802,35 @@ public byte[] findObjectsByPrefix(SwiftObjectPath path, throw new SwiftException("Bad URI: " + dataLocationURI, e); } - return perform("findObjectsByPrefix", uri, new GetMethodProcessor() { - @Override - public byte[] extractResult(GetMethod method) throws IOException { - if (method.getStatusCode() == SC_NOT_FOUND) { - //no result - throw new FileNotFoundException("Not found " + method.getURI()); - } - return method.getResponseBody(); - } + return perform("findObjectsByPrefix", uri, + new GetRequestProcessor() { + @Override + public byte[] extractResult(HttpGet req, HttpResponse resp) + throws IOException { + if (resp.getStatusLine().getStatusCode() == SC_NOT_FOUND) { + //no result + throw new FileNotFoundException("Not found " + req.getURI()); + } + return HttpResponseUtils.getResponseBody(resp); + } - @Override - protected int[] getAllowedStatusCodes() { - return new int[]{ + @Override + protected int[] getAllowedStatusCodes() { + return new int[]{ SC_OK, SC_NOT_FOUND - }; - } + }; + } - @Override - protected void setup(GetMethod method) throws - SwiftInternalStateException { - setHeaders(method, requestHeaders); - } - }); + @Override + protected void setup(HttpGet req) throws SwiftInternalStateException { + setHeaders(req, requestHeaders); + } + }); } /** - * Find objects in a directory + * Find objects in a directory. * * @param path path prefix * @param requestHeaders optional request headers @@ -845,14 +875,14 @@ public byte[] listDeepObjectsInDirectory(SwiftObjectPath path, } /** - * Find objects in a location + * Find objects in a location. * @param location URI * @param requestHeaders optional request headers * @return the body of te response * @throws IOException IO problems */ - private byte[] findObjects(String location, final Header[] requestHeaders) throws - IOException { + private byte[] findObjects(String location, final Header[] requestHeaders) + throws IOException { URI uri; preRemoteCommand("findObjects"); try { @@ -861,30 +891,31 @@ private byte[] findObjects(String location, final Header[] requestHeaders) throw throw new SwiftException("Bad URI: " + location, e); } - return perform("findObjects", uri, new GetMethodProcessor() { - @Override - public byte[] extractResult(GetMethod method) throws IOException { - if (method.getStatusCode() == SC_NOT_FOUND) { - //no result - throw new FileNotFoundException("Not found " + method.getURI()); - } - return method.getResponseBody(); - } + return perform("findObjects", uri, + new GetRequestProcessor() { + @Override + public byte[] extractResult(HttpGet req, HttpResponse resp) + throws IOException { + if (resp.getStatusLine().getStatusCode() == SC_NOT_FOUND) { + //no result + throw new FileNotFoundException("Not found " + req.getURI()); + } + return HttpResponseUtils.getResponseBody(resp); + } - @Override - protected int[] getAllowedStatusCodes() { - return new int[]{ + @Override + protected int[] getAllowedStatusCodes() { + return new int[]{ SC_OK, SC_NOT_FOUND - }; - } + }; + } - @Override - protected void setup(GetMethod method) - throws SwiftInternalStateException { - setHeaders(method, requestHeaders); - } - }); + @Override + protected void setup(HttpGet req) throws SwiftInternalStateException { + setHeaders(req, requestHeaders); + } + }); } /** @@ -903,19 +934,21 @@ public boolean copyObject(SwiftObjectPath src, final SwiftObjectPath dst, preRemoteCommand("copyObject"); - return perform("copy", pathToURI(src), new CopyMethodProcessor() { - @Override - public Boolean extractResult(CopyMethod method) throws IOException { - return method.getStatusCode() != SC_NOT_FOUND; - } + return perform("copy", pathToURI(src), + new CopyRequestProcessor() { + @Override + public Boolean extractResult(CopyRequest req, HttpResponse resp) + throws IOException { + return resp.getStatusLine().getStatusCode() != SC_NOT_FOUND; + } - @Override - protected void setup(CopyMethod method) throws - SwiftInternalStateException { - setHeaders(method, headers); - method.addRequestHeader(HEADER_DESTINATION, dst.toUriPath()); - } - }); + @Override + protected void setup(CopyRequest req) throws + SwiftInternalStateException { + setHeaders(req, headers); + req.addHeader(HEADER_DESTINATION, dst.toUriPath()); + } + }); } /** @@ -936,17 +969,18 @@ public void upload(SwiftObjectPath path, preRemoteCommand("upload"); try { - perform("upload", pathToURI(path), new PutMethodProcessor() { + perform("upload", pathToURI(path), new PutRequestProcessor() { @Override - public byte[] extractResult(PutMethod method) throws IOException { - return method.getResponseBody(); + public byte[] extractResult(HttpPut req, HttpResponse resp) + throws IOException { + return HttpResponseUtils.getResponseBody(resp); } @Override - protected void setup(PutMethod method) throws + protected void setup(HttpPut req) throws SwiftInternalStateException { - method.setRequestEntity(new InputStreamRequestEntity(data, length)); - setHeaders(method, requestHeaders); + req.setEntity(new InputStreamEntity(data, length)); + setHeaders(req, requestHeaders); } }); } finally { @@ -967,22 +1001,23 @@ protected void setup(PutMethod method) throws public boolean delete(SwiftObjectPath path, final Header... requestHeaders) throws IOException { preRemoteCommand("delete"); - return perform("", pathToURI(path), new DeleteMethodProcessor() { + return perform("", pathToURI(path), new DeleteRequestProcessor() { @Override - public Boolean extractResult(DeleteMethod method) throws IOException { - return method.getStatusCode() == SC_NO_CONTENT; + public Boolean extractResult(HttpDelete req, HttpResponse resp) + throws IOException { + return resp.getStatusLine().getStatusCode() == SC_NO_CONTENT; } @Override - protected void setup(DeleteMethod method) throws + protected void setup(HttpDelete req) throws SwiftInternalStateException { - setHeaders(method, requestHeaders); + setHeaders(req, requestHeaders); } }); } /** - * Issue a head request + * Issue a head request. * @param reason reason -used in logs * @param path path to query * @param requestHeaders request header @@ -996,26 +1031,27 @@ public Header[] headRequest(String reason, throws IOException { preRemoteCommand("headRequest: "+ reason); - return perform(reason, pathToURI(path), new HeadMethodProcessor() { - @Override - public Header[] extractResult(HeadMethod method) throws IOException { - if (method.getStatusCode() == SC_NOT_FOUND) { - throw new FileNotFoundException("Not Found " + method.getURI()); - } + return perform(reason, pathToURI(path), + new HeadRequestProcessor() { + @Override + public Header[] extractResult(HttpHead req, HttpResponse resp) + throws IOException { + if (resp.getStatusLine().getStatusCode() == SC_NOT_FOUND) { + throw new FileNotFoundException("Not Found " + req.getURI()); + } + return resp.getAllHeaders(); + } - return method.getResponseHeaders(); - } - - @Override - protected void setup(HeadMethod method) throws - SwiftInternalStateException { - setHeaders(method, requestHeaders); - } - }); + @Override + protected void setup(HttpHead req) throws + SwiftInternalStateException { + setHeaders(req, requestHeaders); + } + }); } /** - * Issue a put request + * Issue a put request. * @param path path * @param requestHeaders optional headers * @return the HTTP response @@ -1025,23 +1061,24 @@ public int putRequest(SwiftObjectPath path, final Header... requestHeaders) throws IOException { preRemoteCommand("putRequest"); - return perform(pathToURI(path), new PutMethodProcessor() { + return perform(pathToURI(path), new PutRequestProcessor() { @Override - public Integer extractResult(PutMethod method) throws IOException { - return method.getStatusCode(); + public Integer extractResult(HttpPut req, HttpResponse resp) + throws IOException { + return resp.getStatusLine().getStatusCode(); } @Override - protected void setup(PutMethod method) throws + protected void setup(HttpPut req) throws SwiftInternalStateException { - setHeaders(method, requestHeaders); + setHeaders(req, requestHeaders); } }); } /** - * Authenticate to Openstack Keystone + * Authenticate to Openstack Keystone. * As well as returning the access token, the member fields {@link #token}, * {@link #endpointURI} and {@link #objectLocationURI} are set up for re-use. *

@@ -1064,7 +1101,8 @@ public AccessToken authenticate() throws IOException { new AuthenticationPost(authenticationRequest)); } - private class AuthenticationPost extends AuthMethodProcessor { + private final class AuthenticationPost extends + AuthRequestProcessor { final AuthenticationRequest authenticationRequest; private AuthenticationPost(AuthenticationRequest authenticationRequest) { @@ -1072,9 +1110,8 @@ private AuthenticationPost(AuthenticationRequest authenticationRequest) { } @Override - protected void setup(AuthPostMethod method) throws IOException { - - method.setRequestEntity(getAuthenticationRequst(authenticationRequest)); + protected void setup(AuthPostRequest req) throws IOException { + req.setEntity(getAuthenticationRequst(authenticationRequest)); } /** @@ -1099,16 +1136,17 @@ protected int[] getAllowedStatusCodes() { } @Override - public AccessToken extractResult(AuthPostMethod method) throws IOException { + public AccessToken extractResult(AuthPostRequest req, HttpResponse resp) + throws IOException { //initial check for failure codes leading to authentication failures - if (method.getStatusCode() == SC_BAD_REQUEST) { + if (resp.getStatusLine().getStatusCode() == SC_BAD_REQUEST) { throw new SwiftAuthenticationFailedException( - authenticationRequest.toString(), "POST", authUri, method); + authenticationRequest.toString(), "POST", authUri, resp); } final AuthenticationResponse access = - JSONUtil.toObject(method.getResponseBodyAsString(), - AuthenticationWrapper.class).getAccess(); + JSONUtil.toObject(HttpResponseUtils.getResponseBodyAsString(resp), + AuthenticationWrapper.class).getAccess(); final List serviceCatalog = access.getServiceCatalog(); //locate the specific service catalog that defines Swift; variations //in the name of this add complexity to the search @@ -1201,14 +1239,15 @@ public AccessToken extractResult(AuthPostMethod method) throws IOException { } } - private StringRequestEntity getAuthenticationRequst(AuthenticationRequest authenticationRequest) - throws IOException { + private StringEntity getAuthenticationRequst( + AuthenticationRequest authenticationRequest) throws IOException { final String data = JSONUtil.toJSON(new AuthenticationRequestWrapper( authenticationRequest)); if (LOG.isDebugEnabled()) { LOG.debug("Authenticating with " + authenticationRequest); } - return toJsonEntity(data); + return new StringEntity(data, ContentType.create("application/json", + "UTF-8")); } /** @@ -1222,7 +1261,7 @@ private synchronized void createDefaultContainer() throws IOException { } /** - * Create a container -if it already exists, do nothing + * Create a container -if it already exists, do nothing. * * @param containerName the container name * @throws IOException IO problems @@ -1271,7 +1310,7 @@ public void createContainer(String containerName) throws IOException { /** * Trigger an initial auth operation if some of the needed - * fields are missing + * fields are missing. * * @throws IOException on problems */ @@ -1314,8 +1353,8 @@ private void preRemoteCommand(String operation) throws IOException { * is invalid * @throws FileNotFoundException a 404 response was returned */ - private R perform(URI uri, - HttpMethodProcessor processor) + private R perform(URI uri, + HttpRequestProcessor processor) throws IOException, SwiftBadRequestException, SwiftInternalStateException, @@ -1341,34 +1380,41 @@ private R perform(URI uri, * is invalid * @throws FileNotFoundException a 404 response was returned */ - private R perform(String reason, - URI uri, - HttpMethodProcessor processor) + private R perform(String reason, URI uri, + HttpRequestProcessor processor) throws IOException, SwiftBadRequestException, SwiftInternalStateException, SwiftInvalidResponseException, FileNotFoundException { checkNotNull(uri); checkNotNull(processor); - final M method = processor.createMethod(uri.toString()); - + final M req = processor.createRequest(uri.toString()); + req.addHeader(HEADER_USER_AGENT, SWIFT_USER_AGENT); //retry policy - HttpMethodParams methodParams = method.getParams(); - methodParams.setParameter(HttpMethodParams.RETRY_HANDLER, - new DefaultHttpMethodRetryHandler( - retryCount, false)); - methodParams.setIntParameter(HttpConnectionParams.CONNECTION_TIMEOUT, - connectTimeout); - methodParams.setSoTimeout(socketTimeout); - method.addRequestHeader(HEADER_USER_AGENT, SWIFT_USER_AGENT); + HttpClientBuilder clientBuilder = HttpClientBuilder.create(); + clientBuilder.setRetryHandler( + new DefaultHttpRequestRetryHandler(retryCount, false)); + RequestConfig.Builder requestConfigBuilder = + RequestConfig.custom().setConnectTimeout(connectTimeout); + if (proxyHost != null) { + requestConfigBuilder.setProxy(new HttpHost(proxyHost, proxyPort)); + } + clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); + clientBuilder.setDefaultSocketConfig( + SocketConfig.custom().setSoTimeout(socketTimeout).build()); Duration duration = new Duration(); boolean success = false; + HttpResponse resp; try { + // client should not be closed in this method because + // the connection can be used later + CloseableHttpClient client = clientBuilder.build(); int statusCode = 0; try { - statusCode = exec(method); + resp = exec(client, req); + statusCode = checkNotNull(resp.getStatusLine().getStatusCode()); } catch (IOException e) { //rethrow with extra diagnostics and wiki links - throw ExceptionDiags.wrapException(uri.toString(), method.getName(), e); + throw ExceptionDiags.wrapException(uri.toString(), req.getMethod(), e); } //look at the response and see if it was valid or not. @@ -1378,23 +1424,23 @@ private R perform(String reason, //validate the allowed status code for this operation int[] allowedStatusCodes = processor.getAllowedStatusCodes(); boolean validResponse = isStatusCodeExpected(statusCode, - allowedStatusCodes); + allowedStatusCodes); if (!validResponse) { - IOException ioe = buildException(uri, method, statusCode); + IOException ioe = buildException(uri, req, resp, statusCode); throw ioe; } - R r = processor.extractResult(method); + R r = processor.extractResult(req, resp); success = true; return r; } catch (IOException e) { //release the connection -always - method.releaseConnection(); + req.releaseConnection(); throw e; } finally { duration.finished(); - durationStats.add(method.getName()+" " + reason, duration, success); + durationStats.add(req.getMethod() + " " + reason, duration, success); } } @@ -1404,94 +1450,91 @@ private R perform(String reason, * {@link SwiftInvalidResponseException}. * * @param uri URI for operation - * @param method operation that failed + * @param resp operation that failed * @param statusCode status code * @param method type * @return an exception to throw */ - private IOException buildException(URI uri, - M method, - int statusCode) { + private IOException buildException( + URI uri, M req, HttpResponse resp, int statusCode) { IOException fault; //log the failure @debug level String errorMessage = String.format("Method %s on %s failed, status code: %d," + " status line: %s", - method.getName(), + req.getMethod(), uri, statusCode, - method.getStatusLine() + resp.getStatusLine() ); if (LOG.isDebugEnabled()) { LOG.debug(errorMessage); } //send the command switch (statusCode) { - case SC_NOT_FOUND: - fault = new FileNotFoundException("Operation " + method.getName() - + " on " + uri); - break; + case SC_NOT_FOUND: + fault = new FileNotFoundException("Operation " + req.getMethod() + + " on " + uri); + break; - case SC_BAD_REQUEST: - //bad HTTP request - fault = new SwiftBadRequestException( - "Bad request against " + uri, - method.getName(), + case SC_BAD_REQUEST: + //bad HTTP request + fault = new SwiftBadRequestException("Bad request against " + uri, + req.getMethod(), uri, resp); + break; + + case SC_REQUESTED_RANGE_NOT_SATISFIABLE: + //out of range + StringBuilder errorText = new StringBuilder( + resp.getStatusLine().getReasonPhrase()); + //get the requested length + Header requestContentLen = req.getFirstHeader(HEADER_CONTENT_LENGTH); + if (requestContentLen != null) { + errorText.append(" requested ").append(requestContentLen.getValue()); + } + //and the result + Header availableContentRange = resp.getFirstHeader(HEADER_CONTENT_RANGE); + + if (availableContentRange != null) { + errorText.append(" available ") + .append(availableContentRange.getValue()); + } + fault = new EOFException(errorText.toString()); + break; + + case SC_UNAUTHORIZED: + //auth failure; should only happen on the second attempt + fault = new SwiftAuthenticationFailedException( + "Operation not authorized- current access token =" + getToken(), + req.getMethod(), uri, - method); - break; + resp); + break; - case SC_REQUESTED_RANGE_NOT_SATISFIABLE: - //out of range - StringBuilder errorText = new StringBuilder(method.getStatusText()); - //get the requested length - Header requestContentLen = method.getRequestHeader(HEADER_CONTENT_LENGTH); - if (requestContentLen!=null) { - errorText.append(" requested ").append(requestContentLen.getValue()); - } - //and the result - Header availableContentRange = method.getResponseHeader( - HEADER_CONTENT_RANGE); - if (availableContentRange != null) { - errorText.append(" available ").append(availableContentRange.getValue()); - } - fault = new EOFException(errorText.toString()); - break; + case SwiftProtocolConstants.SC_TOO_MANY_REQUESTS_429: + case SwiftProtocolConstants.SC_THROTTLED_498: + //response code that may mean the client is being throttled + fault = new SwiftThrottledRequestException( + "Client is being throttled: too many requests", + req.getMethod(), + uri, + resp); + break; - case SC_UNAUTHORIZED: - //auth failure; should only happen on the second attempt - fault = new SwiftAuthenticationFailedException( - "Operation not authorized- current access token =" - + getToken(), - method.getName(), - uri, - method); - break; - - case SwiftProtocolConstants.SC_TOO_MANY_REQUESTS_429: - case SwiftProtocolConstants.SC_THROTTLED_498: - //response code that may mean the client is being throttled - fault = new SwiftThrottledRequestException( - "Client is being throttled: too many requests", - method.getName(), - uri, - method); - break; - - default: - //return a generic invalid HTTP response - fault = new SwiftInvalidResponseException( - errorMessage, - method.getName(), - uri, - method); + default: + //return a generic invalid HTTP response + fault = new SwiftInvalidResponseException( + errorMessage, + req.getMethod(), + uri, + resp); } return fault; } /** - * Exec a GET request and return the input stream of the response + * Exec a GET request and return the input stream of the response. * * @param uri URI to GET * @param requestHeaders request headers @@ -1499,25 +1542,25 @@ private IOException buildException(URI uri, * @throws IOException */ private HttpBodyContent doGet(final URI uri, final Header... requestHeaders) throws IOException { - return perform("", uri, new GetMethodProcessor() { + return perform("", uri, new GetRequestProcessor() { @Override - public HttpBodyContent extractResult(GetMethod method) throws IOException { - return - new HttpBodyContent( - new HttpInputStreamWithRelease(uri, method), method.getResponseContentLength() - ); + public HttpBodyContent extractResult(HttpGet req, HttpResponse resp) + throws IOException { + return new HttpBodyContent( + new HttpInputStreamWithRelease(uri, req, resp), + HttpResponseUtils.getContentLength(resp)); } @Override - protected void setup(GetMethod method) throws + protected void setup(HttpGet req) throws SwiftInternalStateException { - setHeaders(method, requestHeaders); + setHeaders(req, requestHeaders); } }); } /** - * Create an instance against a specific FS URI, + * Create an instance against a specific FS URI. * * @param filesystemURI filesystem to bond to * @param config source of configuration data @@ -1530,25 +1573,6 @@ public static SwiftRestClient getInstance(URI filesystemURI, } - /** - * Convert the (JSON) data to a string request as UTF-8 - * - * @param data data - * @return the data - * @throws SwiftException if for some very unexpected reason it's impossible - * to convert the data to UTF-8. - */ - private static StringRequestEntity toJsonEntity(String data) throws - SwiftException { - StringRequestEntity entity; - try { - entity = new StringRequestEntity(data, "application/json", "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new SwiftException("Could not encode data as UTF-8", e); - } - return entity; - } - /** * Converts Swift path to URI to make request. * This is public for unit testing @@ -1604,76 +1628,72 @@ private URI pathToURI(SwiftObjectPath path) throws SwiftException { } /** - * Add the headers to the method, and the auth token (which must be set + * Add the headers to the method, and the auth token (which must be set). * @param method method to update * @param requestHeaders the list of headers * @throws SwiftInternalStateException not yet authenticated */ - private void setHeaders(HttpMethodBase method, Header[] requestHeaders) + private void setHeaders(HttpUriRequest method, Header[] requestHeaders) throws SwiftInternalStateException { - for (Header header : requestHeaders) { - method.addRequestHeader(header); - } + for (Header header : requestHeaders) { + method.addHeader(header); + } setAuthToken(method, getToken()); } /** - * Set the auth key header of the method to the token ID supplied + * Set the auth key header of the method to the token ID supplied. * * @param method method * @param accessToken access token * @throws SwiftInternalStateException if the client is not yet authenticated */ - private void setAuthToken(HttpMethodBase method, AccessToken accessToken) + private void setAuthToken(HttpUriRequest method, AccessToken accessToken) throws SwiftInternalStateException { checkNotNull(accessToken,"Not authenticated"); - method.addRequestHeader(HEADER_AUTH_KEY, accessToken.getId()); + method.addHeader(HEADER_AUTH_KEY, accessToken.getId()); } /** - * Execute a method in a new HttpClient instance. - * If the auth failed, authenticate then retry the method. + * Execute a method in a new HttpClient instance. If the auth failed, + * authenticate then retry the method. * - * @param method method to exec - * @param Method type + * @param req request to exec + * @param client client to use + * @param Request type * @return the status code * @throws IOException on any failure */ - private int exec(M method) throws IOException { - final HttpClient client = new HttpClient(); - if (proxyHost != null) { - client.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, - new HttpHost(proxyHost, proxyPort)); - } - - int statusCode = execWithDebugOutput(method, client); - + private HttpResponse exec(HttpClient client, M req) + throws IOException { + HttpResponse resp = execWithDebugOutput(req, client); + int statusCode = resp.getStatusLine().getStatusCode(); if ((statusCode == HttpStatus.SC_UNAUTHORIZED || statusCode == HttpStatus.SC_BAD_REQUEST) - && method instanceof AuthPostMethod + && req instanceof AuthPostRequest && !useKeystoneAuthentication) { if (LOG.isDebugEnabled()) { - LOG.debug("Operation failed with status " + method.getStatusCode() + - " attempting keystone auth"); + LOG.debug("Operation failed with status " + statusCode + + " attempting keystone auth"); } //if rackspace key authentication failed - try custom Keystone authentication useKeystoneAuthentication = true; - final AuthPostMethod authentication = (AuthPostMethod) method; + final AuthPostRequest authentication = (AuthPostRequest) req; //replace rackspace auth with keystone one - authentication.setRequestEntity(getAuthenticationRequst(keystoneAuthRequest)); - statusCode = execWithDebugOutput(method, client); + authentication.setEntity(getAuthenticationRequst(keystoneAuthRequest)); + resp = execWithDebugOutput(req, client); } if (statusCode == HttpStatus.SC_UNAUTHORIZED ) { //unauthed -or the auth uri rejected it. - if (method instanceof AuthPostMethod) { + if (req instanceof AuthPostRequest) { //unauth response from the AUTH URI itself. throw new SwiftAuthenticationFailedException(authRequest.toString(), "auth", authUri, - method); + resp); } //any other URL: try again if (LOG.isDebugEnabled()) { @@ -1684,35 +1704,34 @@ private int exec(M method) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Retrying original request"); } - statusCode = execWithDebugOutput(method, client); + resp = execWithDebugOutput(req, client); } - return statusCode; + return resp; } /** - * Execute the request with the request and response logged at debug level - * @param method method to execute + * Execute the request with the request and response logged at debug level. + * @param req request to execute * @param client client to use * @param method type * @return the status code * @throws IOException any failure reported by the HTTP client. */ - private int execWithDebugOutput(M method, - HttpClient client) throws - IOException { + private HttpResponse execWithDebugOutput(M req, + HttpClient client) throws IOException { if (LOG.isDebugEnabled()) { StringBuilder builder = new StringBuilder( - method.getName() + " " + method.getURI() + "\n"); - for (Header header : method.getRequestHeaders()) { + req.getMethod() + " " + req.getURI() + "\n"); + for (Header header : req.getAllHeaders()) { builder.append(header.toString()); } LOG.debug(builder); } - int statusCode = client.executeMethod(method); + HttpResponse resp = client.execute(req); if (LOG.isDebugEnabled()) { - LOG.debug("Status code = " + statusCode); + LOG.debug("Status code = " + resp.getStatusLine().getStatusCode()); } - return statusCode; + return resp; } /** @@ -1741,7 +1760,7 @@ private static T checkNotNull(T reference, String message) throws * * @param status received status * @param expected expected value - * @return true iff status is an element of [expected] + * @return true if status is an element of [expected] */ private boolean isStatusCodeExpected(int status, int... expected) { for (int code : expected) { @@ -1792,7 +1811,7 @@ public String getContainer() { /** * Is this client bound to a location aware Swift blobstore - * -that is, can you query for the location of partitions + * -that is, can you query for the location of partitions? * @return true iff the location of multipart file uploads * can be determined. */ @@ -1809,7 +1828,7 @@ public long getBlocksizeKB() { } /** - * Get the partition size in KB + * Get the partition size in KB. * @return the partition size */ public int getPartSizeKB() { @@ -1817,7 +1836,7 @@ public int getPartSizeKB() { } /** - * Get the buffer size in KB + * Get the buffer size in KB. * @return the buffer size wanted for reads */ public int getBufferSizeKB() { @@ -1849,7 +1868,7 @@ public int getThrottleDelay() { } /** - * Get the current operation statistics + * Get the current operation statistics. * @return a snapshot of the statistics */ diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java index 71d8d824dd..f2ecb0f1da 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java @@ -19,8 +19,9 @@ import com.fasterxml.jackson.databind.type.CollectionType; -import org.apache.commons.httpclient.Header; -import org.apache.commons.httpclient.HttpStatus; +import org.apache.http.Header; +import org.apache.http.HttpStatus; +import org.apache.http.message.BasicHeader; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -166,9 +167,9 @@ public void createManifestForPartUpload(Path path) throws IOException { } swiftRestClient.upload(toObjectPath(path), - new ByteArrayInputStream(new byte[0]), - 0, - new Header(SwiftProtocolConstants.X_OBJECT_MANIFEST, pathString)); + new ByteArrayInputStream(new byte[0]), + 0, + new BasicHeader(SwiftProtocolConstants.X_OBJECT_MANIFEST, pathString)); } /** diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/HttpResponseUtils.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/HttpResponseUtils.java new file mode 100644 index 0000000000..1cc340d83d --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/HttpResponseUtils.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.util; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.util.EncodingUtils; + +import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.HEADER_CONTENT_LENGTH; + +/** + * Utility class for parsing HttpResponse. This class is implemented like + * {@code org.apache.commons.httpclient.HttpMethodBase.java} in httpclient 3.x. + */ +public abstract class HttpResponseUtils { + + /** + * Returns the response body of the HTTPResponse, if any, as an array of bytes. + * If response body is not available or cannot be read, returns null + * + * Note: This will cause the entire response body to be buffered in memory. A + * malicious server may easily exhaust all the VM memory. It is strongly + * recommended, to use getResponseAsStream if the content length of the + * response is unknown or reasonably large. + * + * @param resp HttpResponse + * @return The response body + * @throws IOException If an I/O (transport) problem occurs while obtaining + * the response body. + */ + public static byte[] getResponseBody(HttpResponse resp) throws IOException { + try(InputStream instream = resp.getEntity().getContent()) { + if (instream != null) { + long contentLength = resp.getEntity().getContentLength(); + if (contentLength > Integer.MAX_VALUE) { + //guard integer cast from overflow + throw new IOException("Content too large to be buffered: " + + contentLength +" bytes"); + } + ByteArrayOutputStream outstream = new ByteArrayOutputStream( + contentLength > 0 ? (int) contentLength : 4*1024); + byte[] buffer = new byte[4096]; + int len; + while ((len = instream.read(buffer)) > 0) { + outstream.write(buffer, 0, len); + } + outstream.close(); + return outstream.toByteArray(); + } + } + return null; + } + + /** + * Returns the response body of the HTTPResponse, if any, as a {@link String}. + * If response body is not available or cannot be read, returns null + * The string conversion on the data is done using UTF-8. + * + * Note: This will cause the entire response body to be buffered in memory. A + * malicious server may easily exhaust all the VM memory. It is strongly + * recommended, to use getResponseAsStream if the content length of the + * response is unknown or reasonably large. + * + * @param resp HttpResponse + * @return The response body. + * @throws IOException If an I/O (transport) problem occurs while obtaining + * the response body. + */ + public static String getResponseBodyAsString(HttpResponse resp) + throws IOException { + byte[] rawdata = getResponseBody(resp); + if (rawdata != null) { + return EncodingUtils.getString(rawdata, "UTF-8"); + } else { + return null; + } + } + + /** + * Return the length (in bytes) of the response body, as specified in a + * Content-Length header. + * + *

+ * Return -1 when the content-length is unknown. + *

+ * + * @param resp HttpResponse + * @return content length, if Content-Length header is available. + * 0 indicates that the request has no body. + * If Content-Length header is not present, the method + * returns -1. + */ + public static long getContentLength(HttpResponse resp) { + Header header = resp.getFirstHeader(HEADER_CONTENT_LENGTH); + if (header == null) { + return -1; + } else { + return Long.parseLong(header.getValue()); + } + } +} diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java index 9b223f878f..f344093158 100644 --- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.fs.swift; -import org.apache.commons.httpclient.Header; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataOutputStream; @@ -28,6 +27,7 @@ import org.apache.hadoop.fs.swift.util.SwiftTestUtils; import org.apache.hadoop.fs.swift.util.SwiftUtils; import org.apache.hadoop.io.IOUtils; +import org.apache.http.Header; import org.junit.Test; import org.junit.internal.AssumptionViolatedException; diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java index f24a194cb1..17f616bd9e 100644 --- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java @@ -18,7 +18,6 @@ package org.apache.hadoop.fs.swift.http; -import org.apache.commons.httpclient.Header; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -28,6 +27,7 @@ import org.apache.hadoop.fs.swift.util.DurationStats; import org.apache.hadoop.fs.swift.util.SwiftObjectPath; import org.apache.hadoop.fs.swift.util.SwiftTestUtils; +import org.apache.http.Header; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; diff --git a/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties b/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties index 6aeb41dcdd..a3bb8204f0 100644 --- a/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties +++ b/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties @@ -37,6 +37,3 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n #log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c %x - %m%n" #log4j.logger.org.apache.hadoop.fs.swift=DEBUG - -#crank back on warnings about -1 content length GETs -log4j.logger.org.apache.commons.httpclient.HttpMethodBase=ERROR