From 94e7d49a6dab7e7f4e873dcca67e7fcc98e7e1f8 Mon Sep 17 00:00:00 2001 From: cnauroth Date: Thu, 28 May 2015 12:31:06 -0700 Subject: [PATCH] HADOOP-11959. WASB should configure client side socket timeout in storage client blob request options. Contributed by Ivan Mitic. --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ hadoop-project/pom.xml | 2 +- .../hadoop/fs/azure/AzureNativeFileSystemStore.java | 13 ++----------- .../apache/hadoop/fs/azure/StorageInterface.java | 6 +++--- .../hadoop/fs/azure/StorageInterfaceImpl.java | 4 ++-- .../hadoop/fs/azure/MockStorageInterface.java | 4 ++-- .../azure/TestAzureFileSystemErrorConditions.java | 1 + .../hadoop/fs/azure/TestBlobDataValidation.java | 1 + 8 files changed, 15 insertions(+), 19 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6aa224cb02..aba9087663 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -791,6 +791,9 @@ Release 2.8.0 - UNRELEASED HADOOP-11930. test-patch in offline mode should tell maven to be in offline mode (Sean Busbey via aw) + HADOOP-11959. WASB should configure client side socket timeout in storage + client blob request options. (Ivan Mitic via cnauroth) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index a3470a0e91..16d20580a6 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -920,7 +920,7 @@ com.microsoft.azure azure-storage - 2.0.0 + 2.2.0 diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 3267d8bc42..69bda06c4e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -2434,15 +2434,6 @@ public void rename(String srcKey, String dstKey, boolean acquireLease, // CloudBlobWrapper dstBlob = getBlobReference(dstKey); - // TODO: Remove at the time when we move to Azure Java SDK 1.2+. - // This is the workaround provided by Azure Java SDK team to - // mitigate the issue with un-encoded x-ms-copy-source HTTP - // request header. Azure sdk version before 1.2+ does not encode this - // header what causes all URIs that have special (category "other") - // characters in the URI not to work with startCopyFromBlob when - // specified as source (requests fail with HTTP 403). - URI srcUri = new URI(srcBlob.getUri().toASCIIString()); - // Rename the source blob to the destination blob by copying it to // the destination blob then deleting it. // @@ -2451,7 +2442,7 @@ public void rename(String srcKey, String dstKey, boolean acquireLease, // a more intensive exponential retry policy when the cluster is getting // throttled. try { - dstBlob.startCopyFromBlob(srcUri, null, getInstrumentedContext()); + dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext()); } catch (StorageException se) { if (se.getErrorCode().equals( StorageErrorCode.SERVER_BUSY.toString())) { @@ -2475,7 +2466,7 @@ public void rename(String srcKey, String dstKey, boolean acquireLease, options.setRetryPolicyFactory(new RetryExponentialRetry( copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff, copyBlobMaxRetries)); - dstBlob.startCopyFromBlob(srcUri, options, getInstrumentedContext()); + dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext()); } else { throw se; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java index e89151d709..ce5f749a0d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java @@ -381,8 +381,8 @@ public interface CloudBlobWrapper extends ListBlobItem { * Copies an existing blob's contents, properties, and metadata to this instance of the CloudBlob * class, using the specified operation context. * - * @param source - * A java.net.URI The URI of a source blob. + * @param sourceBlob + * A CloudBlob object that represents the source blob to copy. * @param options * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying * null will use the default request options from the associated service client ( @@ -397,7 +397,7 @@ public interface CloudBlobWrapper extends ListBlobItem { * @throws URISyntaxException * */ - public abstract void startCopyFromBlob(URI source, + public abstract void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options, OperationContext opContext) throws StorageException, URISyntaxException; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java index 90d4d8838a..382ff66345 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java @@ -393,10 +393,10 @@ public CopyState getCopyState() { } @Override - public void startCopyFromBlob(URI source, BlobRequestOptions options, + public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options, OperationContext opContext) throws StorageException, URISyntaxException { - getBlob().startCopyFromBlob(source, + getBlob().startCopyFromBlob(((CloudBlobWrapperImpl)sourceBlob).blob, null, null, options, opContext); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java index cde0e38ef6..9f84f4b39f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java @@ -429,9 +429,9 @@ public void setMetadata(HashMap metadata) { } @Override - public void startCopyFromBlob(URI source, BlobRequestOptions options, + public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options, OperationContext opContext) throws StorageException, URISyntaxException { - backingStore.copy(convertUriToDecodedString(source), convertUriToDecodedString(uri)); + backingStore.copy(convertUriToDecodedString(sourceBlob.getUri()), convertUriToDecodedString(uri)); //TODO: set the backingStore.properties.CopyState and // update azureNativeFileSystemStore.waitForCopyToComplete } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java index ace57dc878..810bcf77de 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java @@ -205,6 +205,7 @@ public void testTransientErrorOnCommitBlockList() throws Exception { @Override public boolean isTargetConnection(HttpURLConnection connection) { return connection.getRequestMethod().equals("PUT") + && connection.getURL().getQuery() != null && connection.getURL().getQuery().contains("blocklist"); } }); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java index 9237ade981..c40b7b52aa 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java @@ -191,6 +191,7 @@ private void checkObtainedMd5(String obtainedMd5) { private static boolean isPutBlock(HttpURLConnection connection) { return connection.getRequestMethod().equals("PUT") + && connection.getURL().getQuery() != null && connection.getURL().getQuery().contains("blockid"); }