big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-19284: [ABFS] Allow "fs.azure.account.hns.enabled" to be set as Account Specific Config (#7062)

Browse Source
This commit is contained in:
Anuj Modi 2024-09-27 10:16:28 +05:30 committed by GitHub
parent 49a495803a
commit 21cdb450ef
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 83 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
View File

@ -450,8 +450,16 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName)
this(rawConfig, accountName, AbfsServiceType.DFS);
}
/**
* Returns the account type as per the user configuration. Gets the account
* specific value if it exists, then looks for an account agnostic value.
* If not configured driver makes additional getAcl call to determine
* the account type during file system initialization.
* @return TRUE/FALSE value if configured, UNKNOWN if not configured.
*/
public Trilean getIsNamespaceEnabledAccount() {
return Trilean.getTrilean(isNamespaceEnabledAccount);
return Trilean.getTrilean(
getString(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount));
}
/**

2
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
View File

@ -405,7 +405,7 @@ private synchronized boolean getNamespaceEnabledInformationFromServer(
}
try {
LOG.debug("Get root ACL status");
getClient().getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext);
getClient(AbfsServiceType.DFS).getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext);
// If getAcl succeeds, namespace is enabled.
isNamespaceEnabled = Trilean.getTrilean(true);
} catch (AbfsRestOperationException ex) {

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
View File

@ -40,6 +40,7 @@
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HTTP_READ_TIMEOUT;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_TOLERATE_CONCURRENT_APPEND;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@ -259,6 +260,9 @@ public void testHttpReadTimeout() throws Exception {
public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs)
throws Exception {
// This is to make sure File System creation goes through before network calls start failing.
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
Configuration conf = this.getRawConfiguration();
// set to small values that will cause timeouts
conf.setInt(AZURE_HTTP_CONNECTION_TIMEOUT, connectionTimeoutMs);

3
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
View File

@ -28,6 +28,7 @@
import org.junit.Test;
import org.mockito.Mockito;
import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
@ -76,7 +77,7 @@ public void testGetAclCallOnHnsConfigAbsence() throws Exception {
getRawConfiguration()));
AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
AbfsClient client = Mockito.spy(fs.getAbfsClient());
Mockito.doReturn(client).when(store).getClient();
Mockito.doReturn(client).when(store).getClient(AbfsServiceType.DFS);
Mockito.doThrow(TrileanConversionException.class)
.when(store)

68
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
View File

@ -28,6 +28,7 @@
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
import org.apache.hadoop.fs.azurebfs.services.AbfsDfsClient;
@ -41,6 +42,8 @@
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
@ -126,6 +129,8 @@ private AzureBlobFileSystem getNewFSWithHnsConf(
Configuration rawConfig = new Configuration();
rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount);
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED,
this.getAccountName()), isNamespaceEnabledAccount);
rawConfig
.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
@ -247,7 +252,7 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
AzureBlobFileSystemStore store = Mockito.spy(getFileSystem().getAbfsStore());
AbfsClient mockClient = mock(AbfsClient.class);
store.setNamespaceEnabled(Trilean.UNKNOWN);
doReturn(mockClient).when(store).getClient();
doReturn(mockClient).when(store).getClient(AbfsServiceType.DFS);
AbfsRestOperationException ex = new AbfsRestOperationException(
statusCode, null, Integer.toString(statusCode), null);
doThrow(ex).when(mockClient).getAclStatus(anyString(), any(TracingContext.class));
@ -271,4 +276,65 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
Mockito.verify(mockClient, times(1))
.getAclStatus(anyString(), any(TracingContext.class));
}
@Test
public void testAccountSpecificConfig() throws Exception {
Configuration rawConfig = new Configuration();
rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED,
this.getAccountName()));
String testAccountName = "testAccount.dfs.core.windows.net";
String otherAccountName = "otherAccount.dfs.core.windows.net";
String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName);
String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName);
// Set both account specific and account agnostic config for test account
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR);
// Assert that account specific config takes precedence
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
// Assert that other account still uses account agnostic config
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, otherUri);
assertFileSystemInitWithExpectedHNSSettings(rawConfig, true);
// Set only the account specific config for test account
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
// Assert that only account specific config is enough for test account
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
// Set only account agnostic config
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, FALSE_STR);
rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName));
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
// Unset both account specific and account agnostic config
rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName));
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
rawConfig.set(AZURE_MAX_IO_RETRIES, "0");
// Assert that file system init fails with UnknownHost exception as getAcl() is needed.
try {
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
} catch (Exception e) {
Assertions.assertThat(e.getCause().getMessage())
.describedAs("getAcl() to determine HNS Nature of account should"
+ "fail with Unknown Host Exception").contains("UnknownHostException");
}
}
private void assertFileSystemInitWithExpectedHNSSettings(
Configuration configuration, boolean expectedIsHnsEnabledValue) throws IOException {
try (AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration)) {
Assertions.assertThat(getIsNamespaceEnabled(fs)).describedAs(
"getIsNamespaceEnabled should return true when the "
+ "account specific config is not set").isEqualTo(expectedIsHnsEnabledValue);
} catch (Exception e) {
throw e;
}
}
}