big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16586. ITestS3GuardFsck, others fails when run using a local metastore. (#1950)

Browse Source
This commit is contained in:
Masatake Iwasaki 2020-05-20 08:47:04 +09:00 committed by GitHub
parent 8f78aeb250
commit 0b7799bf6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardFsck.java
View File

@ -64,8 +64,8 @@ public void setup() throws Exception {
super.setup();
S3AFileSystem fs = getFileSystem();
// These test will fail if no ms
assumeTrue("FS needs to have a metadatastore.",
fs.hasMetadataStore());
assumeTrue("FS needs to have a DynamoDB metadatastore.",
fs.getMetadataStore() instanceof DynamoDBMetadataStore);
assumeTrue("Metadatastore should persist authoritative bit",
metadataStorePersistsAuthoritativeBit(fs.getMetadataStore()));

36
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
View File

@ -21,7 +21,6 @@
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
@ -31,17 +30,23 @@
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.fs.s3a.UnknownStoreException;
import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE_DEFAULT;
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_LOCAL;
import static org.apache.hadoop.fs.s3a.MultipartTestUtils.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getLandsatCSVFile;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@ -57,13 +62,21 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort",
"-force", "-verbose"};
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
removeBaseAndBucketOverrides(conf,
S3_METADATA_STORE_IMPL, S3A_BUCKET_PROBE);
conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_LOCAL);
conf.setInt(S3A_BUCKET_PROBE, S3A_BUCKET_PROBE_DEFAULT);
return conf;
}
@Override
public void setup() throws Exception {
super.setup();
MetadataStore ms = getMetadataStore();
Assume.assumeTrue("Test only applies when a local store is used for S3Guard;"
+ "Store is " + (ms == null ? "none" : ms.toString()),
ms instanceof LocalMetadataStore);
assertTrue("metadata store impl should be LocalMetadataStore.",
getMetadataStore() instanceof LocalMetadataStore);
}
@Test
@ -97,7 +110,6 @@ public void testImportCommand() throws Exception {
.getListing().size());
assertEquals("Expected 2 items: empty directory and a parent directory", 2,
ms.listChildren(parent).getListing().size());
assertTrue(children.isAuthoritative());
}
@Test
@ -165,20 +177,12 @@ public void testImportNoFilesystem() throws Throwable {
@Test
public void testInfoBucketAndRegionNoFS() throws Throwable {
intercept(FileNotFoundException.class,
intercept(UnknownStoreException.class,
() -> run(BucketInfo.NAME, "-meta",
LOCAL_METADATA, "-region",
"any-region", S3A_THIS_BUCKET_DOES_NOT_EXIST));
}
@Test
public void testInitNegativeRead() throws Throwable {
runToFailure(INVALID_ARGUMENT,
Init.NAME, "-meta", LOCAL_METADATA, "-region",
"eu-west-1",
READ_FLAG, "-10");
}
@Test
public void testInit() throws Throwable {
run(Init.NAME,