big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18861. ABFS: Fix failing tests for CPK (#5979)

Browse Source 
Contributed by Anmol Asrani
This commit is contained in:
Anmol Asrani 2023-10-09 22:10:15 +05:30 committed by GitHub
parent 666af58700
commit 9c621fcea7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
View File

@ -109,10 +109,14 @@ public ITestCustomerProvidedKey() throws Exception {
.getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); .getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
} }
private String getFileName() throws IOException {
return path("/" + methodName.getMethodName()).toUri().getPath();
}
@Test @Test
public void testReadWithCPK() throws Exception { public void testReadWithCPK() throws Exception {
final AzureBlobFileSystem fs = getAbfs(true); final AzureBlobFileSystem fs = getAbfs(true);
String fileName = path("/" + methodName.getMethodName()).toString(); String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE); createFileAndGetContent(fs, fileName, FILE_SIZE);
AbfsClient abfsClient = fs.getAbfsClient(); AbfsClient abfsClient = fs.getAbfsClient();
@ -162,7 +166,7 @@ public void testReadWithCPK() throws Exception {
@Test @Test
public void testReadWithoutCPK() throws Exception { public void testReadWithoutCPK() throws Exception {
final AzureBlobFileSystem fs = getAbfs(false); final AzureBlobFileSystem fs = getAbfs(false);
String fileName = path("/" + methodName.getMethodName()).toString(); String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE); createFileAndGetContent(fs, fileName, FILE_SIZE);
AbfsClient abfsClient = fs.getAbfsClient(); AbfsClient abfsClient = fs.getAbfsClient();
@ -201,7 +205,7 @@ public void testReadWithoutCPK() throws Exception {
@Test @Test
public void testAppendWithCPK() throws Exception { public void testAppendWithCPK() throws Exception {
final AzureBlobFileSystem fs = getAbfs(true); final AzureBlobFileSystem fs = getAbfs(true);
final String fileName = path("/" + methodName.getMethodName()).toString(); final String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE); createFileAndGetContent(fs, fileName, FILE_SIZE);
// Trying to append with correct CPK headers // Trying to append with correct CPK headers
@ -246,7 +250,7 @@ public void testAppendWithCPK() throws Exception {
@Test @Test
public void testAppendWithoutCPK() throws Exception { public void testAppendWithoutCPK() throws Exception {
final AzureBlobFileSystem fs = getAbfs(false); final AzureBlobFileSystem fs = getAbfs(false);
final String fileName = path("/" + methodName.getMethodName()).toString(); final String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE); createFileAndGetContent(fs, fileName, FILE_SIZE);
// Trying to append without CPK headers // Trying to append without CPK headers
@ -282,7 +286,7 @@ public void testAppendWithoutCPK() throws Exception {
@Test @Test
public void testSetGetXAttr() throws Exception { public void testSetGetXAttr() throws Exception {
final AzureBlobFileSystem fs = getAbfs(true); final AzureBlobFileSystem fs = getAbfs(true);
final String fileName = path(methodName.getMethodName()).toString(); final String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE); createFileAndGetContent(fs, fileName, FILE_SIZE);
String valSent = "testValue"; String valSent = "testValue";
@ -416,7 +420,7 @@ public void testListPathWithoutCPK() throws Exception {
private void testListPath(final boolean isWithCPK) throws Exception { private void testListPath(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final Path testPath = path("/" + methodName.getMethodName()); final Path testPath = path("/" + methodName.getMethodName());
String testDirName = testPath.toString(); String testDirName = testPath.toUri().getPath();
fs.mkdirs(testPath); fs.mkdirs(testPath);
createFileAndGetContent(fs, testDirName + "/aaa", FILE_SIZE); createFileAndGetContent(fs, testDirName + "/aaa", FILE_SIZE);
createFileAndGetContent(fs, testDirName + "/bbb", FILE_SIZE); createFileAndGetContent(fs, testDirName + "/bbb", FILE_SIZE);
@ -475,8 +479,7 @@ public void testCreatePathWithoutCPK() throws Exception {
private void testCreatePath(final boolean isWithCPK) throws Exception { private void testCreatePath(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
createFileAndGetContent(fs, testFileName, FILE_SIZE); createFileAndGetContent(fs, testFileName, FILE_SIZE);
AbfsClient abfsClient = fs.getAbfsClient(); AbfsClient abfsClient = fs.getAbfsClient();
@ -519,8 +522,7 @@ public void testRenamePathWithoutCPK() throws Exception {
private void testRenamePath(final boolean isWithCPK) throws Exception { private void testRenamePath(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
createFileAndGetContent(fs, testFileName, FILE_SIZE); createFileAndGetContent(fs, testFileName, FILE_SIZE);
FileStatus fileStatusBeforeRename = fs FileStatus fileStatusBeforeRename = fs
@ -556,8 +558,7 @@ public void testFlushWithoutCPK() throws Exception {
private void testFlush(final boolean isWithCPK) throws Exception { private void testFlush(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
fs.create(new Path(testFileName)).close(); fs.create(new Path(testFileName)).close();
AbfsClient abfsClient = fs.getAbfsClient(); AbfsClient abfsClient = fs.getAbfsClient();
String expectedCPKSha = getCPKSha(fs); String expectedCPKSha = getCPKSha(fs);
@ -617,8 +618,7 @@ public void testSetPathPropertiesWithoutCPK() throws Exception {
private void testSetPathProperties(final boolean isWithCPK) throws Exception { private void testSetPathProperties(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
createFileAndGetContent(fs, testFileName, FILE_SIZE); createFileAndGetContent(fs, testFileName, FILE_SIZE);
AbfsClient abfsClient = fs.getAbfsClient(); AbfsClient abfsClient = fs.getAbfsClient();
@ -648,8 +648,7 @@ public void testGetPathStatusFileWithoutCPK() throws Exception {
private void testGetPathStatusFile(final boolean isWithCPK) throws Exception { private void testGetPathStatusFile(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
createFileAndGetContent(fs, testFileName, FILE_SIZE); createFileAndGetContent(fs, testFileName, FILE_SIZE);
AbfsClient abfsClient = fs.getAbfsClient(); AbfsClient abfsClient = fs.getAbfsClient();
@ -686,8 +685,7 @@ public void testDeletePathWithoutCPK() throws Exception {
private void testDeletePath(final boolean isWithCPK) throws Exception { private void testDeletePath(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
createFileAndGetContent(fs, testFileName, FILE_SIZE); createFileAndGetContent(fs, testFileName, FILE_SIZE);
FileStatus[] listStatuses = fs.listStatus(new Path(testFileName)); FileStatus[] listStatuses = fs.listStatus(new Path(testFileName));
@ -717,8 +715,7 @@ public void testSetPermissionWithoutCPK() throws Exception {
private void testSetPermission(final boolean isWithCPK) throws Exception { private void testSetPermission(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
Assume.assumeTrue(fs.getIsNamespaceEnabled(getTestTracingContext(fs, false))); Assume.assumeTrue(fs.getIsNamespaceEnabled(getTestTracingContext(fs, false)));
createFileAndGetContent(fs, testFileName, FILE_SIZE); createFileAndGetContent(fs, testFileName, FILE_SIZE);
AbfsClient abfsClient = fs.getAbfsClient(); AbfsClient abfsClient = fs.getAbfsClient();
@ -743,8 +740,7 @@ public void testSetAclWithoutCPK() throws Exception {
private void testSetAcl(final boolean isWithCPK) throws Exception { private void testSetAcl(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
TracingContext tracingContext = getTestTracingContext(fs, false); TracingContext tracingContext = getTestTracingContext(fs, false);
Assume.assumeTrue(fs.getIsNamespaceEnabled(tracingContext)); Assume.assumeTrue(fs.getIsNamespaceEnabled(tracingContext));
createFileAndGetContent(fs, testFileName, FILE_SIZE); createFileAndGetContent(fs, testFileName, FILE_SIZE);
@ -773,8 +769,7 @@ public void testGetAclWithoutCPK() throws Exception {
private void testGetAcl(final boolean isWithCPK) throws Exception { private void testGetAcl(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
TracingContext tracingContext = getTestTracingContext(fs, false); TracingContext tracingContext = getTestTracingContext(fs, false);
Assume.assumeTrue(fs.getIsNamespaceEnabled(tracingContext)); Assume.assumeTrue(fs.getIsNamespaceEnabled(tracingContext));
createFileAndGetContent(fs, testFileName, FILE_SIZE); createFileAndGetContent(fs, testFileName, FILE_SIZE);
@ -804,8 +799,7 @@ private void testCheckAccess(final boolean isWithCPK) throws Exception {
getAuthType() == AuthType.OAuth); getAuthType() == AuthType.OAuth);
final AzureBlobFileSystem fs = getAbfs(isWithCPK); final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName()) final String testFileName = getFileName();
.toString();
fs.create(new Path(testFileName)).close(); fs.create(new Path(testFileName)).close();
AbfsClient abfsClient = fs.getAbfsClient(); AbfsClient abfsClient = fs.getAbfsClient();
AbfsRestOperation abfsRestOperation = abfsClient AbfsRestOperation abfsRestOperation = abfsClient

1
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java
View File

@ -382,6 +382,7 @@ public void testConfigPropNotFound() throws Throwable {
for (String key : CONFIG_KEYS) { for (String key : CONFIG_KEYS) {
setAuthConfig(abfsConf, true, AuthType.OAuth); setAuthConfig(abfsConf, true, AuthType.OAuth);
abfsConf.unset(key);
abfsConf.unset(key + "." + accountName); abfsConf.unset(key + "." + accountName);
testMissingConfigKey(abfsConf, key); testMissingConfigKey(abfsConf, key);
} }