diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 628faa3d3a..14cd75a8ca 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -653,6 +653,9 @@ Release 2.7.0 - UNRELEASED HADOOP-11648. Set DomainSocketWatcher thread name explicitly. (Liang Xie via ozawa) + HADOOP-11642. Upgrade azure sdk version from 0.6.0 to 2.0.0. + (Shashank Khandelwal and Ivan Mitic via cnauroth) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 2c0f03a5f1..a6127c7630 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -902,9 +902,9 @@ - com.microsoft.windowsazure.storage - microsoft-windowsazure-storage-sdk - 0.6.0 + com.microsoft.azure + azure-storage + 2.0.0 diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index d39dd769e1..e9b3af775f 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -140,12 +140,13 @@ httpclient compile - + - com.microsoft.windowsazure.storage - microsoft-windowsazure-storage-sdk + com.microsoft.azure + azure-storage compile + com.google.guava diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index c0c03b3fce..b664fe760c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -65,23 +65,23 @@ import org.mortbay.util.ajax.JSON; import com.google.common.annotations.VisibleForTesting; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RetryExponentialRetry; -import com.microsoft.windowsazure.storage.RetryNoRetry; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageCredentialsAccountAndKey; -import com.microsoft.windowsazure.storage.StorageCredentialsSharedAccessSignature; -import com.microsoft.windowsazure.storage.StorageErrorCode; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobListingDetails; -import com.microsoft.windowsazure.storage.blob.BlobProperties; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.blob.CopyStatus; -import com.microsoft.windowsazure.storage.blob.DeleteSnapshotsOption; -import com.microsoft.windowsazure.storage.blob.ListBlobItem; -import com.microsoft.windowsazure.storage.core.Utility; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RetryExponentialRetry; +import com.microsoft.azure.storage.RetryNoRetry; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; +import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; +import com.microsoft.azure.storage.StorageErrorCode; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobListingDetails; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CopyStatus; +import com.microsoft.azure.storage.blob.DeleteSnapshotsOption; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.core.Utility; /** * Core implementation of Windows Azure Filesystem for Hadoop. @@ -2543,7 +2543,8 @@ public void updateFolderLastModifiedTime(String key, Date lastModified, try { checkContainer(ContainerAccessType.ReadThenWrite); CloudBlobWrapper blob = getBlobReference(key); - blob.getProperties().setLastModified(lastModified); + //setLastModified function is not available in 2.0.0 version. blob.uploadProperties automatically updates last modified + //timestamp to current time blob.uploadProperties(getInstrumentedContext(), folderLease); } catch (Exception e) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 0248b85a28..e39b37d5e4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -74,11 +74,11 @@ import org.codehaus.jackson.map.ObjectMapper; import com.google.common.annotations.VisibleForTesting; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.core.*; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.core.*; /** *

diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java index ad11aacbcf..9a316a51bd 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java @@ -20,7 +20,7 @@ import java.nio.ByteBuffer; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.BlobRequestOptions; /** * Constants and helper methods for ASV's custom data format in page blobs. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java index 62b47ee4f4..468ac65d76 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java @@ -33,10 +33,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.PageRange; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.PageRange; /** * An input stream that reads file data from a page blob stored diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java index 4d1d5c8e6a..2b8846c7f7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java @@ -44,10 +44,10 @@ import org.apache.hadoop.conf.Configuration; import com.google.common.annotations.VisibleForTesting; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudPageBlob; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudPageBlob; /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java index bda6006d60..06f32ce361 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java @@ -22,9 +22,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlob; import java.util.concurrent.atomic.AtomicInteger; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java index d18a14406e..a9e3df907f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java @@ -25,11 +25,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RequestResult; -import com.microsoft.windowsazure.storage.ResponseReceivedEvent; -import com.microsoft.windowsazure.storage.SendingRequestEvent; -import com.microsoft.windowsazure.storage.StorageEvent; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RequestResult; +import com.microsoft.azure.storage.ResponseReceivedEvent; +import com.microsoft.azure.storage.SendingRequestEvent; +import com.microsoft.azure.storage.StorageEvent; /* * Self throttling is implemented by hooking into send & response callbacks diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java index 18f173ebbf..4d564d5338 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java @@ -25,12 +25,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.Constants.HeaderConstants; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.SendingRequestEvent; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageEvent; -import com.microsoft.windowsazure.storage.StorageException; +import com.microsoft.azure.storage.Constants.HeaderConstants; +import com.microsoft.azure.storage.core.StorageCredentialsHelper; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.SendingRequestEvent; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageEvent; +import com.microsoft.azure.storage.StorageException; /** * Manages the lifetime of binding on the operation contexts to intercept send @@ -146,7 +147,8 @@ && isOutOfBandIoAllowed()) { try { // Sign the request. GET's have no payload so the content length is // zero. - getCredentials().signBlobAndQueueRequest(urlConnection, -1L, getOperationContext()); + StorageCredentialsHelper.signBlobAndQueueRequest(getCredentials(), + urlConnection, -1L, getOperationContext()); } catch (InvalidKeyException e) { // Log invalid key exception to track signing error before the send // fails. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java index 8d0229d8cf..91928a2904 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java @@ -29,18 +29,18 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RetryPolicyFactory; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobListingDetails; -import com.microsoft.windowsazure.storage.blob.BlobProperties; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.blob.CopyState; -import com.microsoft.windowsazure.storage.blob.ListBlobItem; -import com.microsoft.windowsazure.storage.blob.PageRange; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RetryPolicyFactory; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobListingDetails; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CopyState; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.blob.PageRange; /** * This is a very thin layer over the methods exposed by the Windows Azure diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java index e44823c82f..2120536109 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java @@ -30,26 +30,26 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RetryPolicyFactory; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.StorageUri; -import com.microsoft.windowsazure.storage.blob.BlobListingDetails; -import com.microsoft.windowsazure.storage.blob.BlobProperties; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.blob.CloudBlobClient; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlobDirectory; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; -import com.microsoft.windowsazure.storage.blob.CloudPageBlob; -import com.microsoft.windowsazure.storage.blob.CopyState; -import com.microsoft.windowsazure.storage.blob.DeleteSnapshotsOption; -import com.microsoft.windowsazure.storage.blob.ListBlobItem; -import com.microsoft.windowsazure.storage.blob.PageRange; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RetryPolicyFactory; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.StorageUri; +import com.microsoft.azure.storage.blob.BlobListingDetails; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.CloudPageBlob; +import com.microsoft.azure.storage.blob.CopyState; +import com.microsoft.azure.storage.blob.DeleteSnapshotsOption; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.blob.PageRange; /** * A real implementation of the Azure interaction layer that just redirects @@ -61,12 +61,14 @@ class StorageInterfaceImpl extends StorageInterface { @Override public void setRetryPolicyFactory(final RetryPolicyFactory retryPolicyFactory) { - serviceClient.setRetryPolicyFactory(retryPolicyFactory); + serviceClient.getDefaultRequestOptions().setRetryPolicyFactory( + retryPolicyFactory); } @Override public void setTimeoutInMs(int timeoutInMs) { - serviceClient.setTimeoutInMs(timeoutInMs); + serviceClient.getDefaultRequestOptions().setTimeoutIntervalInMs( + timeoutInMs); } @Override diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java index d33e8c447f..dc23354e7b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java @@ -24,10 +24,10 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RequestResult; -import com.microsoft.windowsazure.storage.ResponseReceivedEvent; -import com.microsoft.windowsazure.storage.StorageEvent; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RequestResult; +import com.microsoft.azure.storage.ResponseReceivedEvent; +import com.microsoft.azure.storage.StorageEvent; /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java index 676adb941c..de503bf190 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java @@ -24,11 +24,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.Constants.HeaderConstants; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RequestResult; -import com.microsoft.windowsazure.storage.ResponseReceivedEvent; -import com.microsoft.windowsazure.storage.StorageEvent; +import com.microsoft.azure.storage.Constants.HeaderConstants; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RequestResult; +import com.microsoft.azure.storage.ResponseReceivedEvent; +import com.microsoft.azure.storage.StorageEvent; /** diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java index b8ff912b4e..635c024185 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java @@ -40,20 +40,20 @@ import org.apache.hadoop.metrics2.MetricsTag; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageCredentialsAccountAndKey; -import com.microsoft.windowsazure.storage.StorageCredentialsAnonymous; -import com.microsoft.windowsazure.storage.blob.BlobContainerPermissions; -import com.microsoft.windowsazure.storage.blob.BlobContainerPublicAccessType; -import com.microsoft.windowsazure.storage.blob.BlobOutputStream; -import com.microsoft.windowsazure.storage.blob.CloudBlobClient; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; -import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPermissions; -import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy; -import com.microsoft.windowsazure.storage.core.Base64; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; +import com.microsoft.azure.storage.StorageCredentialsAnonymous; +import com.microsoft.azure.storage.blob.BlobContainerPermissions; +import com.microsoft.azure.storage.blob.BlobContainerPublicAccessType; +import com.microsoft.azure.storage.blob.BlobOutputStream; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions; +import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy; +import com.microsoft.azure.storage.core.Base64; /** * Helper class to create WASB file systems backed by either a mock in-memory diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java index 047ea1b8fa..c51c05bd53 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java @@ -22,10 +22,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Calendar; +import java.util.Date; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -35,21 +37,21 @@ import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.lang.NotImplementedException; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RetryPolicyFactory; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.StorageUri; -import com.microsoft.windowsazure.storage.blob.BlobListingDetails; -import com.microsoft.windowsazure.storage.blob.BlobProperties; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlobDirectory; -import com.microsoft.windowsazure.storage.blob.CopyState; -import com.microsoft.windowsazure.storage.blob.ListBlobItem; -import com.microsoft.windowsazure.storage.blob.PageRange; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RetryPolicyFactory; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.StorageUri; +import com.microsoft.azure.storage.blob.BlobListingDetails; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import com.microsoft.azure.storage.blob.CopyState; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.blob.PageRange; import javax.ws.rs.core.UriBuilder; import javax.ws.rs.core.UriBuilderException; @@ -357,18 +359,42 @@ protected MockCloudBlobWrapper(URI uri, HashMap metadata, this.uri = uri; this.metadata = metadata; this.properties = new BlobProperties(); - this.properties.setLength(length); - this.properties.setLastModified( - Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime()); + + this.properties=updateLastModifed(this.properties); + this.properties=updateLength(this.properties,length); } - + + protected BlobProperties updateLastModifed(BlobProperties properties){ + try{ + Method setLastModified =properties.getClass(). + getDeclaredMethod("setLastModified", Date.class); + setLastModified.setAccessible(true); + setLastModified.invoke(this.properties, + Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime()); + }catch(Exception e){ + throw new RuntimeException(e); + } + return properties; + } + + protected BlobProperties updateLength(BlobProperties properties,int length) { + try{ + Method setLength =properties.getClass(). + getDeclaredMethod("setLength", long.class); + setLength.setAccessible(true); + setLength.invoke(this.properties, length); + }catch (Exception e){ + throw new RuntimeException(e); + } + return properties; + } + protected void refreshProperties(boolean getMetadata) { if (backingStore.exists(convertUriToDecodedString(uri))) { byte[] content = backingStore.getContent(convertUriToDecodedString(uri)); properties = new BlobProperties(); - properties.setLength(content.length); - properties.setLastModified( - Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime()); + this.properties=updateLastModifed(this.properties); + this.properties=updateLength(this.properties, content.length); if (getMetadata) { metadata = backingStore.getMetadata(convertUriToDecodedString(uri)); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java index 01cf71399f..9ce6cc9b81 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java @@ -58,9 +58,9 @@ import org.apache.hadoop.fs.azure.AzureException; import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlob; /* * Tests the Native Azure file system (WASB) against an actual blob store if diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java index febb6053ba..ace57dc878 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java @@ -37,9 +37,9 @@ import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext; import org.junit.Test; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.SendingRequestEvent; -import com.microsoft.windowsazure.storage.StorageEvent; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.SendingRequestEvent; +import com.microsoft.azure.storage.StorageEvent; public class TestAzureFileSystemErrorConditions { private static final int ALL_THREE_FILE_SIZE = 1024; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java index 25bd338e2c..9237ade981 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java @@ -39,16 +39,16 @@ import org.junit.After; import org.junit.Test; -import com.microsoft.windowsazure.storage.Constants; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.ResponseReceivedEvent; -import com.microsoft.windowsazure.storage.StorageErrorCodeStrings; -import com.microsoft.windowsazure.storage.StorageEvent; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlockEntry; -import com.microsoft.windowsazure.storage.blob.BlockSearchMode; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; -import com.microsoft.windowsazure.storage.core.Base64; +import com.microsoft.azure.storage.Constants; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.ResponseReceivedEvent; +import com.microsoft.azure.storage.StorageErrorCodeStrings; +import com.microsoft.azure.storage.StorageEvent; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlockEntry; +import com.microsoft.azure.storage.blob.BlockSearchMode; +import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.core.Base64; /** * Test that we do proper data integrity validation with MD5 checks as diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java index 727f540efa..56ec881243 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java @@ -32,9 +32,9 @@ import org.junit.After; import org.junit.Test; -import com.microsoft.windowsazure.storage.blob.BlobOutputStream; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.BlobOutputStream; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; /** * Tests that WASB creates containers only if needed. diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java index 9ac67e70f8..60b01c616e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java @@ -29,8 +29,8 @@ import org.junit.Before; import org.junit.Test; -import com.microsoft.windowsazure.storage.blob.BlobOutputStream; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.BlobOutputStream; +import com.microsoft.azure.storage.blob.CloudBlockBlob; public class TestOutOfBandAzureBlobOperationsLive { private FileSystem fs; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java index 0360e32317..a4ca6fdda4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java @@ -42,8 +42,8 @@ import org.junit.Assert; import org.junit.Test; -import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; -import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; public class TestWasbUriAndConfiguration {