big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17714 ABFS: testBlobBackCompatibility, testRandomRead & WasbAbfsCompatibility tests fail when triggered with default configs (#3035)

Browse Source
This commit is contained in:
snehavarma 2021-06-13 23:52:29 +05:30 committed by GitHub
parent 5970c632d4
commit 35e4c31fff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
View File

@ -149,7 +149,7 @@ public void setup() throws Exception {
if (!isIPAddress if (!isIPAddress
&& (abfsConfig.getAuthType(accountName) != AuthType.SAS) && (abfsConfig.getAuthType(accountName) != AuthType.SAS)
&& !abfs.getIsNamespaceEnabled()) { && !abfs.getIsNamespaceEnabled()) {
final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl())); final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl(), abfsConfig.isHttpsAlwaysUsed()));
final AzureNativeFileSystemStore azureNativeFileSystemStore = final AzureNativeFileSystemStore azureNativeFileSystemStore =
new AzureNativeFileSystemStore(); new AzureNativeFileSystemStore();
@ -355,13 +355,13 @@ protected void touch(Path path) throws IOException {
protected static String wasbUrlToAbfsUrl(final String wasbUrl) { protected static String wasbUrlToAbfsUrl(final String wasbUrl) {
return convertTestUrls( return convertTestUrls(
wasbUrl, FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX, wasbUrl, FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX,
FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX); FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX, false);
} }
protected static String abfsUrlToWasbUrl(final String abfsUrl) { protected static String abfsUrlToWasbUrl(final String abfsUrl, final boolean isAlwaysHttpsUsed) {
return convertTestUrls( return convertTestUrls(
abfsUrl, FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX, abfsUrl, FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX,
FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX); FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX, isAlwaysHttpsUsed);
} }
private static String convertTestUrls( private static String convertTestUrls(
@ -371,15 +371,17 @@ private static String convertTestUrls(
final String fromDnsPrefix, final String fromDnsPrefix,
final String toNonSecureScheme, final String toNonSecureScheme,
final String toSecureScheme, final String toSecureScheme,
final String toDnsPrefix) { final String toDnsPrefix,
final boolean isAlwaysHttpsUsed) {
String data = null; String data = null;
if (url.startsWith(fromNonSecureScheme + "://")) { if (url.startsWith(fromNonSecureScheme + "://") && isAlwaysHttpsUsed) {
data = url.replace(fromNonSecureScheme + "://", toSecureScheme + "://");
} else if (url.startsWith(fromNonSecureScheme + "://")) {
data = url.replace(fromNonSecureScheme + "://", toNonSecureScheme + "://"); data = url.replace(fromNonSecureScheme + "://", toNonSecureScheme + "://");
} else if (url.startsWith(fromSecureScheme + "://")) { } else if (url.startsWith(fromSecureScheme + "://")) {
data = url.replace(fromSecureScheme + "://", toSecureScheme + "://"); data = url.replace(fromSecureScheme + "://", toSecureScheme + "://");
} }
if (data != null) { if (data != null) {
data = data.replace("." + fromDnsPrefix + ".", data = data.replace("." + fromDnsPrefix + ".",
"." + toDnsPrefix + "."); "." + toDnsPrefix + ".");

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
View File

@ -74,6 +74,12 @@ private String getBlobConnectionString() {
+ ";AccountName=" + this.getAccountName().split("\\.")[0] + ";AccountName=" + this.getAccountName().split("\\.")[0]
+ ";AccountKey=" + this.getAccountKey(); + ";AccountKey=" + this.getAccountKey();
} }
else if (this.getConfiguration().isHttpsAlwaysUsed()) {
connectionString = "DefaultEndpointsProtocol=https;BlobEndpoint=https://"
+ this.getAccountName().replaceFirst("\\.dfs\\.", ".blob.")
+ ";AccountName=" + this.getAccountName().split("\\.")[0]
+ ";AccountKey=" + this.getAccountKey();
}
else { else {
connectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://" connectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://"
+ this.getAccountName().replaceFirst("\\.dfs\\.", ".blob.") + this.getAccountName().replaceFirst("\\.dfs\\.", ".blob.")

2
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
View File

@ -159,7 +159,7 @@ public void testUrlConversion(){
String abfsUrl = "abfs://abcde-1111-1111-1111-1111@xxxx.dfs.xxx.xxx.xxxx.xxxx"; String abfsUrl = "abfs://abcde-1111-1111-1111-1111@xxxx.dfs.xxx.xxx.xxxx.xxxx";
String wabsUrl = "wasb://abcde-1111-1111-1111-1111@xxxx.blob.xxx.xxx.xxxx.xxxx"; String wabsUrl = "wasb://abcde-1111-1111-1111-1111@xxxx.blob.xxx.xxx.xxxx.xxxx";
assertEquals(abfsUrl, wasbUrlToAbfsUrl(wabsUrl)); assertEquals(abfsUrl, wasbUrlToAbfsUrl(wabsUrl));
assertEquals(wabsUrl, abfsUrlToWasbUrl(abfsUrl)); assertEquals(wabsUrl, abfsUrlToWasbUrl(abfsUrl, false));
} }
@Test @Test