HADOOP-19284: [ABFS] Allow "fs.azure.account.hns.enabled" to be set as Account Specific Config (#7062)
This commit is contained in:
parent
49a495803a
commit
21cdb450ef
@ -450,8 +450,16 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName)
|
|||||||
this(rawConfig, accountName, AbfsServiceType.DFS);
|
this(rawConfig, accountName, AbfsServiceType.DFS);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the account type as per the user configuration. Gets the account
|
||||||
|
* specific value if it exists, then looks for an account agnostic value.
|
||||||
|
* If not configured driver makes additional getAcl call to determine
|
||||||
|
* the account type during file system initialization.
|
||||||
|
* @return TRUE/FALSE value if configured, UNKNOWN if not configured.
|
||||||
|
*/
|
||||||
public Trilean getIsNamespaceEnabledAccount() {
|
public Trilean getIsNamespaceEnabledAccount() {
|
||||||
return Trilean.getTrilean(isNamespaceEnabledAccount);
|
return Trilean.getTrilean(
|
||||||
|
getString(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -405,7 +405,7 @@ private synchronized boolean getNamespaceEnabledInformationFromServer(
|
|||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
LOG.debug("Get root ACL status");
|
LOG.debug("Get root ACL status");
|
||||||
getClient().getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext);
|
getClient(AbfsServiceType.DFS).getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext);
|
||||||
// If getAcl succeeds, namespace is enabled.
|
// If getAcl succeeds, namespace is enabled.
|
||||||
isNamespaceEnabled = Trilean.getTrilean(true);
|
isNamespaceEnabled = Trilean.getTrilean(true);
|
||||||
} catch (AbfsRestOperationException ex) {
|
} catch (AbfsRestOperationException ex) {
|
||||||
|
@ -40,6 +40,7 @@
|
|||||||
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HTTP_READ_TIMEOUT;
|
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HTTP_READ_TIMEOUT;
|
||||||
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
|
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
|
||||||
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_TOLERATE_CONCURRENT_APPEND;
|
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_TOLERATE_CONCURRENT_APPEND;
|
||||||
|
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
|
||||||
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
|
||||||
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists;
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists;
|
||||||
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
||||||
@ -259,6 +260,9 @@ public void testHttpReadTimeout() throws Exception {
|
|||||||
|
|
||||||
public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs)
|
public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
// This is to make sure File System creation goes through before network calls start failing.
|
||||||
|
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
|
||||||
|
|
||||||
Configuration conf = this.getRawConfiguration();
|
Configuration conf = this.getRawConfiguration();
|
||||||
// set to small values that will cause timeouts
|
// set to small values that will cause timeouts
|
||||||
conf.setInt(AZURE_HTTP_CONNECTION_TIMEOUT, connectionTimeoutMs);
|
conf.setInt(AZURE_HTTP_CONNECTION_TIMEOUT, connectionTimeoutMs);
|
||||||
|
@ -28,6 +28,7 @@
|
|||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.mockito.Mockito;
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType;
|
||||||
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
|
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
|
||||||
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
|
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
|
||||||
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
|
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
|
||||||
@ -76,7 +77,7 @@ public void testGetAclCallOnHnsConfigAbsence() throws Exception {
|
|||||||
getRawConfiguration()));
|
getRawConfiguration()));
|
||||||
AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
|
AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
|
||||||
AbfsClient client = Mockito.spy(fs.getAbfsClient());
|
AbfsClient client = Mockito.spy(fs.getAbfsClient());
|
||||||
Mockito.doReturn(client).when(store).getClient();
|
Mockito.doReturn(client).when(store).getClient(AbfsServiceType.DFS);
|
||||||
|
|
||||||
Mockito.doThrow(TrileanConversionException.class)
|
Mockito.doThrow(TrileanConversionException.class)
|
||||||
.when(store)
|
.when(store)
|
||||||
|
@ -28,6 +28,7 @@
|
|||||||
|
|
||||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType;
|
||||||
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
|
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
|
||||||
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
|
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
|
||||||
import org.apache.hadoop.fs.azurebfs.services.AbfsDfsClient;
|
import org.apache.hadoop.fs.azurebfs.services.AbfsDfsClient;
|
||||||
@ -41,6 +42,8 @@
|
|||||||
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
|
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
|
||||||
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
|
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
|
||||||
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
|
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
|
||||||
|
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
|
||||||
|
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
import static org.mockito.Mockito.doReturn;
|
import static org.mockito.Mockito.doReturn;
|
||||||
@ -126,6 +129,8 @@ private AzureBlobFileSystem getNewFSWithHnsConf(
|
|||||||
Configuration rawConfig = new Configuration();
|
Configuration rawConfig = new Configuration();
|
||||||
rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
|
rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
|
||||||
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount);
|
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount);
|
||||||
|
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED,
|
||||||
|
this.getAccountName()), isNamespaceEnabledAccount);
|
||||||
rawConfig
|
rawConfig
|
||||||
.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
|
.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
|
||||||
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
|
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
|
||||||
@ -247,7 +252,7 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
|
|||||||
AzureBlobFileSystemStore store = Mockito.spy(getFileSystem().getAbfsStore());
|
AzureBlobFileSystemStore store = Mockito.spy(getFileSystem().getAbfsStore());
|
||||||
AbfsClient mockClient = mock(AbfsClient.class);
|
AbfsClient mockClient = mock(AbfsClient.class);
|
||||||
store.setNamespaceEnabled(Trilean.UNKNOWN);
|
store.setNamespaceEnabled(Trilean.UNKNOWN);
|
||||||
doReturn(mockClient).when(store).getClient();
|
doReturn(mockClient).when(store).getClient(AbfsServiceType.DFS);
|
||||||
AbfsRestOperationException ex = new AbfsRestOperationException(
|
AbfsRestOperationException ex = new AbfsRestOperationException(
|
||||||
statusCode, null, Integer.toString(statusCode), null);
|
statusCode, null, Integer.toString(statusCode), null);
|
||||||
doThrow(ex).when(mockClient).getAclStatus(anyString(), any(TracingContext.class));
|
doThrow(ex).when(mockClient).getAclStatus(anyString(), any(TracingContext.class));
|
||||||
@ -271,4 +276,65 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
|
|||||||
Mockito.verify(mockClient, times(1))
|
Mockito.verify(mockClient, times(1))
|
||||||
.getAclStatus(anyString(), any(TracingContext.class));
|
.getAclStatus(anyString(), any(TracingContext.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAccountSpecificConfig() throws Exception {
|
||||||
|
Configuration rawConfig = new Configuration();
|
||||||
|
rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
|
||||||
|
rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
|
||||||
|
rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED,
|
||||||
|
this.getAccountName()));
|
||||||
|
String testAccountName = "testAccount.dfs.core.windows.net";
|
||||||
|
String otherAccountName = "otherAccount.dfs.core.windows.net";
|
||||||
|
String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName);
|
||||||
|
String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName);
|
||||||
|
|
||||||
|
// Set both account specific and account agnostic config for test account
|
||||||
|
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
|
||||||
|
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR);
|
||||||
|
// Assert that account specific config takes precedence
|
||||||
|
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
|
||||||
|
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
|
||||||
|
// Assert that other account still uses account agnostic config
|
||||||
|
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, otherUri);
|
||||||
|
assertFileSystemInitWithExpectedHNSSettings(rawConfig, true);
|
||||||
|
|
||||||
|
// Set only the account specific config for test account
|
||||||
|
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
|
||||||
|
rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
|
||||||
|
// Assert that only account specific config is enough for test account
|
||||||
|
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
|
||||||
|
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
|
||||||
|
|
||||||
|
// Set only account agnostic config
|
||||||
|
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, FALSE_STR);
|
||||||
|
rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName));
|
||||||
|
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
|
||||||
|
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
|
||||||
|
|
||||||
|
// Unset both account specific and account agnostic config
|
||||||
|
rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
|
||||||
|
rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName));
|
||||||
|
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
|
||||||
|
rawConfig.set(AZURE_MAX_IO_RETRIES, "0");
|
||||||
|
// Assert that file system init fails with UnknownHost exception as getAcl() is needed.
|
||||||
|
try {
|
||||||
|
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
|
||||||
|
} catch (Exception e) {
|
||||||
|
Assertions.assertThat(e.getCause().getMessage())
|
||||||
|
.describedAs("getAcl() to determine HNS Nature of account should"
|
||||||
|
+ "fail with Unknown Host Exception").contains("UnknownHostException");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertFileSystemInitWithExpectedHNSSettings(
|
||||||
|
Configuration configuration, boolean expectedIsHnsEnabledValue) throws IOException {
|
||||||
|
try (AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration)) {
|
||||||
|
Assertions.assertThat(getIsNamespaceEnabled(fs)).describedAs(
|
||||||
|
"getIsNamespaceEnabled should return true when the "
|
||||||
|
+ "account specific config is not set").isEqualTo(expectedIsHnsEnabledValue);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user