HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI
This commit is contained in:
parent
375224edeb
commit
9a8edb0aed
@ -643,6 +643,20 @@ public String getScheme() {
|
|||||||
return "wasb";
|
return "wasb";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If fs.azure.override.canonical.service.name is set as true, return URI of
|
||||||
|
* the WASB filesystem, otherwise use the default implementation.
|
||||||
|
*
|
||||||
|
* @return a service string that uniquely identifies this file system
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public String getCanonicalServiceName() {
|
||||||
|
if (returnUriAsCanonicalServiceName) {
|
||||||
|
return getUri().toString();
|
||||||
|
}
|
||||||
|
return super.getCanonicalServiceName();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p>
|
* <p>
|
||||||
@ -726,6 +740,11 @@ public String getScheme() {
|
|||||||
*/
|
*/
|
||||||
public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME = "fs.azure.enable.append.support";
|
public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME = "fs.azure.enable.append.support";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Property to override canonical service name with filesystem's URI.
|
||||||
|
*/
|
||||||
|
public static final String RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME = "fs.azure.override.canonical.service.name";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The configuration property to set number of threads to be used for rename operation.
|
* The configuration property to set number of threads to be used for rename operation.
|
||||||
*/
|
*/
|
||||||
@ -1192,6 +1211,7 @@ private void restoreKey() throws IOException {
|
|||||||
// A counter to create unique (within-process) names for my metrics sources.
|
// A counter to create unique (within-process) names for my metrics sources.
|
||||||
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
|
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
|
||||||
private boolean appendSupportEnabled = false;
|
private boolean appendSupportEnabled = false;
|
||||||
|
private boolean returnUriAsCanonicalServiceName = false;
|
||||||
private DelegationTokenAuthenticatedURL authURL;
|
private DelegationTokenAuthenticatedURL authURL;
|
||||||
private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
|
private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
|
||||||
private String credServiceUrl;
|
private String credServiceUrl;
|
||||||
@ -1389,6 +1409,8 @@ public void initialize(URI uri, Configuration conf)
|
|||||||
if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
|
if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
|
||||||
this.wasbDelegationTokenManager = new RemoteWasbDelegationTokenManager(conf);
|
this.wasbDelegationTokenManager = new RemoteWasbDelegationTokenManager(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.returnUriAsCanonicalServiceName = conf.getBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -19,6 +19,7 @@
|
|||||||
package org.apache.hadoop.fs.azure;
|
package org.apache.hadoop.fs.azure;
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
|
||||||
|
import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME;
|
||||||
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
||||||
import static org.junit.Assume.assumeFalse;
|
import static org.junit.Assume.assumeFalse;
|
||||||
import static org.junit.Assume.assumeNotNull;
|
import static org.junit.Assume.assumeNotNull;
|
||||||
@ -44,7 +45,6 @@
|
|||||||
import org.apache.hadoop.fs.FileContext;
|
import org.apache.hadoop.fs.FileContext;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.UnsupportedFileSystemException;
|
|
||||||
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
|
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
|
||||||
import org.apache.hadoop.test.GenericTestUtils;
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
|
||||||
@ -640,4 +640,32 @@ public void testUserAgentConfig() throws Exception {
|
|||||||
FileSystem.closeAll();
|
FileSystem.closeAll();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCanonicalServiceName() throws Exception {
|
||||||
|
AzureBlobStorageTestAccount testAccount = AzureBlobStorageTestAccount.createMock();
|
||||||
|
Configuration conf = testAccount.getFileSystem().getConf();
|
||||||
|
String authority = testAccount.getFileSystem().getUri().getAuthority();
|
||||||
|
URI defaultUri = new URI("wasbs", authority, null, null, null);
|
||||||
|
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
|
||||||
|
|
||||||
|
try {
|
||||||
|
FileSystem fs0 = FileSystem.get(conf);
|
||||||
|
// Default getCanonicalServiceName() will try to resolve the host to IP,
|
||||||
|
// because the mock container does not exist, this call is expected to fail.
|
||||||
|
intercept(IllegalArgumentException.class,
|
||||||
|
"java.net.UnknownHostException",
|
||||||
|
() -> {
|
||||||
|
fs0.getCanonicalServiceName();
|
||||||
|
});
|
||||||
|
|
||||||
|
conf.setBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, true);
|
||||||
|
FileSystem fs1 = FileSystem.newInstance(defaultUri, conf);
|
||||||
|
Assert.assertEquals("getCanonicalServiceName() should return URI",
|
||||||
|
fs1.getUri().toString(), fs1.getCanonicalServiceName());
|
||||||
|
} finally {
|
||||||
|
testAccount.cleanup();
|
||||||
|
FileSystem.closeAll();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user