big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11959. WASB should configure client side socket timeout in storage client blob request options. Contributed by Ivan Mitic.

Browse Source
This commit is contained in:
cnauroth 2015-05-28 12:31:06 -07:00
parent 7ebe80ec12
commit 94e7d49a6d
8 changed files with 15 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -791,6 +791,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11930. test-patch in offline mode should tell maven to be in
offline mode (Sean Busbey via aw)
HADOOP-11959. WASB should configure client side socket timeout in storage
client blob request options. (Ivan Mitic via cnauroth)
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES

2
hadoop-project/pom.xml
View File

@ -920,7 +920,7 @@
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
<version>2.0.0</version>
<version>2.2.0</version>
</dependency>
<dependency>

13
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
View File

@ -2434,15 +2434,6 @@ public void rename(String srcKey, String dstKey, boolean acquireLease,
//
CloudBlobWrapper dstBlob = getBlobReference(dstKey);
// TODO: Remove at the time when we move to Azure Java SDK 1.2+.
// This is the workaround provided by Azure Java SDK team to
// mitigate the issue with un-encoded x-ms-copy-source HTTP
// request header. Azure sdk version before 1.2+ does not encode this
// header what causes all URIs that have special (category "other")
// characters in the URI not to work with startCopyFromBlob when
// specified as source (requests fail with HTTP 403).
URI srcUri = new URI(srcBlob.getUri().toASCIIString());
// Rename the source blob to the destination blob by copying it to
// the destination blob then deleting it.
//
@ -2451,7 +2442,7 @@ public void rename(String srcKey, String dstKey, boolean acquireLease,
// a more intensive exponential retry policy when the cluster is getting
// throttled.
try {
dstBlob.startCopyFromBlob(srcUri, null, getInstrumentedContext());
dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext());
} catch (StorageException se) {
if (se.getErrorCode().equals(
StorageErrorCode.SERVER_BUSY.toString())) {
@ -2475,7 +2466,7 @@ public void rename(String srcKey, String dstKey, boolean acquireLease,
options.setRetryPolicyFactory(new RetryExponentialRetry(
copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff,
copyBlobMaxRetries));
dstBlob.startCopyFromBlob(srcUri, options, getInstrumentedContext());
dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext());
} else {
throw se;
}

6
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
View File

@ -381,8 +381,8 @@ public interface CloudBlobWrapper extends ListBlobItem {
* Copies an existing blob's contents, properties, and metadata to this instance of the <code>CloudBlob</code>
* class, using the specified operation context.
*
* @param source
* A <code>java.net.URI</code> The URI of a source blob.
* @param sourceBlob
* A <code>CloudBlob</code> object that represents the source blob to copy.
* @param options
* A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
* <code>null</code> will use the default request options from the associated service client (
@ -397,7 +397,7 @@ public interface CloudBlobWrapper extends ListBlobItem {
* @throws URISyntaxException
*
*/
public abstract void startCopyFromBlob(URI source,
public abstract void startCopyFromBlob(CloudBlobWrapper sourceBlob,
BlobRequestOptions options, OperationContext opContext)
throws StorageException, URISyntaxException;

4
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
View File

@ -393,10 +393,10 @@ public CopyState getCopyState() {
}
@Override
public void startCopyFromBlob(URI source, BlobRequestOptions options,
public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
OperationContext opContext)
throws StorageException, URISyntaxException {
getBlob().startCopyFromBlob(source,
getBlob().startCopyFromBlob(((CloudBlobWrapperImpl)sourceBlob).blob,
null, null, options, opContext);
}

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
View File

@ -429,9 +429,9 @@ public void setMetadata(HashMap<String, String> metadata) {
}
@Override
public void startCopyFromBlob(URI source, BlobRequestOptions options,
public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
OperationContext opContext) throws StorageException, URISyntaxException {
backingStore.copy(convertUriToDecodedString(source), convertUriToDecodedString(uri));
backingStore.copy(convertUriToDecodedString(sourceBlob.getUri()), convertUriToDecodedString(uri));
//TODO: set the backingStore.properties.CopyState and
// update azureNativeFileSystemStore.waitForCopyToComplete
}

1
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
View File

@ -205,6 +205,7 @@ public void testTransientErrorOnCommitBlockList() throws Exception {
@Override
public boolean isTargetConnection(HttpURLConnection connection) {
return connection.getRequestMethod().equals("PUT")
&& connection.getURL().getQuery() != null
&& connection.getURL().getQuery().contains("blocklist");
}
});

1
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
View File

@ -191,6 +191,7 @@ private void checkObtainedMd5(String obtainedMd5) {
private static boolean isPutBlock(HttpURLConnection connection) {
return connection.getRequestMethod().equals("PUT")
&& connection.getURL().getQuery() != null
&& connection.getURL().getQuery().contains("blockid");
}