diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 6f794d0d4f..382cd7fbd6 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -133,7 +133,8 @@ public void setup() throws Exception { //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem. createFileSystem(); - if (!isIPAddress && authType == AuthType.SharedKey) { + // Only live account without namespace support can run ABFS&WASB compatibility tests + if (!isIPAddress && !abfs.getIsNamespaceEnabled()) { final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl())); final AzureNativeFileSystemStore azureNativeFileSystemStore = new AzureNativeFileSystemStore(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java index 22d4990cc4..5ac16b45a4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java @@ -26,7 +26,6 @@ import org.junit.Assume; import org.junit.Test; -import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -38,14 +37,13 @@ public class ITestAzureBlobFileSystemBackCompat extends public ITestAzureBlobFileSystemBackCompat() throws Exception { super(); - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } @Test public void testBlobBackCompat() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); - // test only valid for non-namespace enabled account - Assume.assumeFalse(fs.getIsNamespaceEnabled()); + Assume.assumeFalse("This test does not support namespace enabled account", + this.getFileSystem().getIsNamespaceEnabled()); String storageConnectionString = getBlobConnectionString(); CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java index 707a1452d5..f514696330 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java @@ -21,8 +21,6 @@ import java.io.IOException; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.azurebfs.services.AuthType; -import org.junit.Ignore; import org.junit.Test; import org.apache.hadoop.fs.FileStatus; @@ -37,6 +35,7 @@ public class ITestAzureBlobFileSystemFileStatus extends private static final String DEFAULT_FILE_PERMISSION_VALUE = "640"; private static final String DEFAULT_DIR_PERMISSION_VALUE = "750"; private static final String DEFAULT_UMASK_VALUE = "027"; + private static final String FULL_PERMISSION = "777"; private static final Path TEST_FILE = new Path("testFile"); private static final Path TEST_FOLDER = new Path("testDir"); @@ -54,7 +53,6 @@ public void testEnsureStatusWorksForRoot() throws Exception { assertEquals("root listing", 0, rootls.length); } - @Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.") @Test public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); @@ -69,13 +67,16 @@ private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name, String errorInStatus = "error in " + fileStatus + " from " + fs; - // When running with Oauth, the owner and group info retrieved from server will be digit ids. - if (this.getAuthType() != AuthType.OAuth && !fs.isSecureScheme()) { + if (!fs.getIsNamespaceEnabled()) { assertEquals(errorInStatus + ": owner", fs.getOwnerUser(), fileStatus.getOwner()); assertEquals(errorInStatus + ": group", fs.getOwnerUserPrimaryGroup(), fileStatus.getGroup()); + assertEquals(new FsPermission(FULL_PERMISSION), fileStatus.getPermission()); } else { + // When running with namespace enabled account, + // the owner and group info retrieved from server will be digit ids. + // hence skip the owner and group validation if (isDir) { assertEquals(errorInStatus + ": permission", new FsPermission(DEFAULT_DIR_PERMISSION_VALUE), fileStatus.getPermission()); @@ -88,7 +89,6 @@ private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name, return fileStatus; } - @Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.") @Test public void testFolderStatusPermissionsAndOwnerAndGroup() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java index 9d1738857c..3c21525549 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java @@ -25,14 +25,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.azurebfs.services.AuthType; /** * Test finalize() method when "fs.abfs.impl.disable.cache" is enabled. */ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{ static final String DISABLE_ABFS_CACHE_KEY = "fs.abfs.impl.disable.cache"; - static final String DISABLE_ABFSSS_CACHE_KEY = "fs.abfss.impl.disable.cache"; + static final String DISABLE_ABFSS_CACHE_KEY = "fs.abfss.impl.disable.cache"; public ITestAzureBlobFileSystemFinalize() throws Exception { super(); @@ -42,9 +41,8 @@ public ITestAzureBlobFileSystemFinalize() throws Exception { public void testFinalize() throws Exception { // Disable the cache for filesystem to make sure there is no reference. Configuration rawConfig = this.getRawConfiguration(); - rawConfig.setBoolean( - this.getAuthType() == AuthType.SharedKey ? DISABLE_ABFS_CACHE_KEY : DISABLE_ABFSSS_CACHE_KEY, - true); + rawConfig.setBoolean(DISABLE_ABFS_CACHE_KEY, true); + rawConfig.setBoolean(DISABLE_ABFSS_CACHE_KEY, true); AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java index 23a1ab5bb7..d60cae868f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; import org.hamcrest.core.IsEqual; import org.hamcrest.core.IsNot; -import org.junit.Assume; import org.junit.Test; import org.apache.hadoop.fs.FSDataInputStream; @@ -42,8 +41,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.services.AuthType; - /** * Test flush operation. * This class cannot be run in parallel test mode--check comments in @@ -221,8 +218,6 @@ public void testFlushWithFlushDisabled() throws Exception { } private void testFlush(boolean flushEnabled) throws Exception { - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); - final AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem(); // Simulate setting "fs.azure.enable.flush" to true or false diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java index 5f08721ada..5c4b87b0d2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java @@ -46,8 +46,7 @@ public void teardown() { @Test (expected = FileNotFoundException.class) public void ensureFilesystemWillNotBeCreatedIfCreationConfigIsNotSet() throws Exception { - super.setup(); - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = this.createFileSystem(); FileStatus[] fileStatuses = fs.listStatus(new Path("/")); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java index 768f4bbc5d..e5f64b5f2c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java @@ -34,7 +34,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; -import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.contract.ContractTestUtils; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -67,7 +66,6 @@ public class ITestAzureBlobFileSystemRandomRead extends public ITestAzureBlobFileSystemRandomRead() throws Exception { super(); - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } @Test @@ -98,6 +96,8 @@ public void testBasicRead() throws Exception { */ @Test public void testRandomRead() throws Exception { + Assume.assumeFalse("This test does not support namespace enabled account", + this.getFileSystem().getIsNamespaceEnabled()); assumeHugeFileExists(); try ( FSDataInputStream inputStreamV1 @@ -413,6 +413,8 @@ public void testSequentialReadAfterReverseSeekPerformance() @Test public void testRandomReadPerformance() throws Exception { + Assume.assumeFalse("This test does not support namespace enabled account", + this.getFileSystem().getIsNamespaceEnabled()); createTestFile(); assumeHugeFileExists(); @@ -523,11 +525,7 @@ private static double toMbps(long bytes, long milliseconds) { } private void createTestFile() throws Exception { - final AzureBlobFileSystem abFs = this.getFileSystem(); - // test only valid for non-namespace enabled account - Assume.assumeFalse(abFs.getIsNamespaceEnabled()); - FileSystem fs = this.getWasbFileSystem(); - + final AzureBlobFileSystem fs = this.getFileSystem(); if (fs.exists(TEST_FILE_PATH)) { FileStatus status = fs.getFileStatus(TEST_FILE_PATH); if (status.getLen() >= TEST_FILE_SIZE) { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java index fdf9788c36..e8f845c9a3 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java @@ -31,7 +31,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.fs.azurebfs.services.AuthType; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDeleted; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory; @@ -51,7 +50,6 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { public ITestWasbAbfsCompatibility() throws Exception { Assume.assumeFalse("Emulator is not supported", isIPAddress()); - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } @Test @@ -59,7 +57,8 @@ public void testListFileStatus() throws Exception { // crate file using abfs AzureBlobFileSystem fs = getFileSystem(); // test only valid for non-namespace enabled account - Assume.assumeFalse(fs.getIsNamespaceEnabled()); + Assume.assumeFalse("Namespace enabled account does not support this test,", + fs.getIsNamespaceEnabled()); NativeAzureFileSystem wasb = getWasbFileSystem(); @@ -93,7 +92,8 @@ public void testReadFile() throws Exception { AzureBlobFileSystem abfs = getFileSystem(); // test only valid for non-namespace enabled account - Assume.assumeFalse(abfs.getIsNamespaceEnabled()); + Assume.assumeFalse("Namespace enabled account does not support this test", + abfs.getIsNamespaceEnabled()); NativeAzureFileSystem wasb = getWasbFileSystem(); @@ -132,7 +132,8 @@ public void testDir() throws Exception { AzureBlobFileSystem abfs = getFileSystem(); // test only valid for non-namespace enabled account - Assume.assumeFalse(abfs.getIsNamespaceEnabled()); + Assume.assumeFalse("Namespace enabled account does not support this test", + abfs.getIsNamespaceEnabled()); NativeAzureFileSystem wasb = getWasbFileSystem(); @@ -166,7 +167,8 @@ public void testSetWorkingDirectory() throws Exception { //create folders AzureBlobFileSystem abfs = getFileSystem(); // test only valid for non-namespace enabled account - Assume.assumeFalse(abfs.getIsNamespaceEnabled()); + Assume.assumeFalse("Namespace enabled account does not support this test", + abfs.getIsNamespaceEnabled()); NativeAzureFileSystem wasb = getWasbFileSystem();