big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15825. ABFS: Enable some tests for namespace not enabled account using OAuth.

Browse Source 
Contributed by Da Zhou.
This commit is contained in:
Steve Loughran 2018-10-09 20:02:12 +01:00
parent a05bd1288c
commit bd50fa956b
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
8 changed files with 27 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
View File

@ -133,7 +133,8 @@ public void setup() throws Exception {
//Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem.
createFileSystem();
if (!isIPAddress && authType == AuthType.SharedKey) {
// Only live account without namespace support can run ABFS&WASB compatibility tests
if (!isIPAddress && !abfs.getIsNamespaceEnabled()) {
final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl()));
final AzureNativeFileSystemStore azureNativeFileSystemStore =
new AzureNativeFileSystemStore();

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
View File

@ -26,7 +26,6 @@
import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@ -38,14 +37,13 @@ public class ITestAzureBlobFileSystemBackCompat extends
public ITestAzureBlobFileSystemBackCompat() throws Exception {
super();
Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
}
@Test
public void testBlobBackCompat() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
// test only valid for non-namespace enabled account
Assume.assumeFalse(fs.getIsNamespaceEnabled());
Assume.assumeFalse("This test does not support namespace enabled account",
this.getFileSystem().getIsNamespaceEnabled());
String storageConnectionString = getBlobConnectionString();
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();

12
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
View File

@ -21,8 +21,6 @@
import java.io.IOException;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.junit.Ignore;
import org.junit.Test;
import org.apache.hadoop.fs.FileStatus;
@ -37,6 +35,7 @@ public class ITestAzureBlobFileSystemFileStatus extends
private static final String DEFAULT_FILE_PERMISSION_VALUE = "640";
private static final String DEFAULT_DIR_PERMISSION_VALUE = "750";
private static final String DEFAULT_UMASK_VALUE = "027";
private static final String FULL_PERMISSION = "777";
private static final Path TEST_FILE = new Path("testFile");
private static final Path TEST_FOLDER = new Path("testDir");
@ -54,7 +53,6 @@ public void testEnsureStatusWorksForRoot() throws Exception {
assertEquals("root listing", 0, rootls.length);
}
@Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.")
@Test
public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
@ -69,13 +67,16 @@ private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name,
String errorInStatus = "error in " + fileStatus + " from " + fs;
// When running with Oauth, the owner and group info retrieved from server will be digit ids.
if (this.getAuthType() != AuthType.OAuth && !fs.isSecureScheme()) {
if (!fs.getIsNamespaceEnabled()) {
assertEquals(errorInStatus + ": owner",
fs.getOwnerUser(), fileStatus.getOwner());
assertEquals(errorInStatus + ": group",
fs.getOwnerUserPrimaryGroup(), fileStatus.getGroup());
assertEquals(new FsPermission(FULL_PERMISSION), fileStatus.getPermission());
} else {
// When running with namespace enabled account,
// the owner and group info retrieved from server will be digit ids.
// hence skip the owner and group validation
if (isDir) {
assertEquals(errorInStatus + ": permission",
new FsPermission(DEFAULT_DIR_PERMISSION_VALUE), fileStatus.getPermission());
@ -88,7 +89,6 @@ private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name,
return fileStatus;
}
@Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.")
@Test
public void testFolderStatusPermissionsAndOwnerAndGroup() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();

8
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java
View File

@ -25,14 +25,13 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
/**
* Test finalize() method when "fs.abfs.impl.disable.cache" is enabled.
*/
public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{
static final String DISABLE_ABFS_CACHE_KEY = "fs.abfs.impl.disable.cache";
static final String DISABLE_ABFSSS_CACHE_KEY = "fs.abfss.impl.disable.cache";
static final String DISABLE_ABFSS_CACHE_KEY = "fs.abfss.impl.disable.cache";
public ITestAzureBlobFileSystemFinalize() throws Exception {
super();
@ -42,9 +41,8 @@ public ITestAzureBlobFileSystemFinalize() throws Exception {
public void testFinalize() throws Exception {
// Disable the cache for filesystem to make sure there is no reference.
Configuration rawConfig = this.getRawConfiguration();
rawConfig.setBoolean(
this.getAuthType() == AuthType.SharedKey ? DISABLE_ABFS_CACHE_KEY : DISABLE_ABFSSS_CACHE_KEY,
true);
rawConfig.setBoolean(DISABLE_ABFS_CACHE_KEY, true);
rawConfig.setBoolean(DISABLE_ABFSS_CACHE_KEY, true);
AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig);

5
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
View File

@ -33,7 +33,6 @@
import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
import org.hamcrest.core.IsEqual;
import org.hamcrest.core.IsNot;
import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.fs.FSDataInputStream;
@ -42,8 +41,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
/**
* Test flush operation.
* This class cannot be run in parallel test mode--check comments in
@ -221,8 +218,6 @@ public void testFlushWithFlushDisabled() throws Exception {
}
private void testFlush(boolean flushEnabled) throws Exception {
Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
final AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem();
// Simulate setting "fs.azure.enable.flush" to true or false

3
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
View File

@ -46,8 +46,7 @@ public void teardown() {
@Test (expected = FileNotFoundException.class)
public void ensureFilesystemWillNotBeCreatedIfCreationConfigIsNotSet() throws Exception {
super.setup();
final AzureBlobFileSystem fs = this.getFileSystem();
final AzureBlobFileSystem fs = this.createFileSystem();
FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
}
}

12
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
View File

@ -34,7 +34,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@ -67,7 +66,6 @@ public class ITestAzureBlobFileSystemRandomRead extends
public ITestAzureBlobFileSystemRandomRead() throws Exception {
super();
Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
}
@Test
@ -98,6 +96,8 @@ public void testBasicRead() throws Exception {
*/
@Test
public void testRandomRead() throws Exception {
Assume.assumeFalse("This test does not support namespace enabled account",
this.getFileSystem().getIsNamespaceEnabled());
assumeHugeFileExists();
try (
FSDataInputStream inputStreamV1
@ -413,6 +413,8 @@ public void testSequentialReadAfterReverseSeekPerformance()
@Test
public void testRandomReadPerformance() throws Exception {
Assume.assumeFalse("This test does not support namespace enabled account",
this.getFileSystem().getIsNamespaceEnabled());
createTestFile();
assumeHugeFileExists();
@ -523,11 +525,7 @@ private static double toMbps(long bytes, long milliseconds) {
}
private void createTestFile() throws Exception {
final AzureBlobFileSystem abFs = this.getFileSystem();
// test only valid for non-namespace enabled account
Assume.assumeFalse(abFs.getIsNamespaceEnabled());
FileSystem fs = this.getWasbFileSystem();
final AzureBlobFileSystem fs = this.getFileSystem();
if (fs.exists(TEST_FILE_PATH)) {
FileStatus status = fs.getFileStatus(TEST_FILE_PATH);
if (status.getLen() >= TEST_FILE_SIZE) {

14
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
View File

@ -31,7 +31,6 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDeleted;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
@ -51,7 +50,6 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
public ITestWasbAbfsCompatibility() throws Exception {
Assume.assumeFalse("Emulator is not supported", isIPAddress());
Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
}
@Test
@ -59,7 +57,8 @@ public void testListFileStatus() throws Exception {
// crate file using abfs
AzureBlobFileSystem fs = getFileSystem();
// test only valid for non-namespace enabled account
Assume.assumeFalse(fs.getIsNamespaceEnabled());
Assume.assumeFalse("Namespace enabled account does not support this test,",
fs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
@ -93,7 +92,8 @@ public void testReadFile() throws Exception {
AzureBlobFileSystem abfs = getFileSystem();
// test only valid for non-namespace enabled account
Assume.assumeFalse(abfs.getIsNamespaceEnabled());
Assume.assumeFalse("Namespace enabled account does not support this test",
abfs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
@ -132,7 +132,8 @@ public void testDir() throws Exception {
AzureBlobFileSystem abfs = getFileSystem();
// test only valid for non-namespace enabled account
Assume.assumeFalse(abfs.getIsNamespaceEnabled());
Assume.assumeFalse("Namespace enabled account does not support this test",
abfs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
@ -166,7 +167,8 @@ public void testSetWorkingDirectory() throws Exception {
//create folders
AzureBlobFileSystem abfs = getFileSystem();
// test only valid for non-namespace enabled account
Assume.assumeFalse(abfs.getIsNamespaceEnabled());
Assume.assumeFalse("Namespace enabled account does not support this test",
abfs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();