HADOOP-16586. ITestS3GuardFsck, others fails when run using a local metastore. (#1950)

This commit is contained in:
Masatake Iwasaki 2020-05-20 08:47:04 +09:00 committed by GitHub
parent 8f78aeb250
commit 0b7799bf6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 18 deletions

View File

@ -64,8 +64,8 @@ public void setup() throws Exception {
super.setup(); super.setup();
S3AFileSystem fs = getFileSystem(); S3AFileSystem fs = getFileSystem();
// These test will fail if no ms // These test will fail if no ms
assumeTrue("FS needs to have a metadatastore.", assumeTrue("FS needs to have a DynamoDB metadatastore.",
fs.hasMetadataStore()); fs.getMetadataStore() instanceof DynamoDBMetadataStore);
assumeTrue("Metadatastore should persist authoritative bit", assumeTrue("Metadatastore should persist authoritative bit",
metadataStorePersistsAuthoritativeBit(fs.getMetadataStore())); metadataStorePersistsAuthoritativeBit(fs.getMetadataStore()));

View File

@ -21,7 +21,6 @@
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.util.ArrayList; import java.util.ArrayList;
@ -31,17 +30,23 @@
import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.junit.Assume;
import org.junit.Test; import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.S3AFileStatus; import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.Tristate; import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.fs.s3a.UnknownStoreException;
import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE_DEFAULT;
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_LOCAL;
import static org.apache.hadoop.fs.s3a.MultipartTestUtils.*; import static org.apache.hadoop.fs.s3a.MultipartTestUtils.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getLandsatCSVFile; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getLandsatCSVFile;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@ -57,13 +62,21 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort", private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort",
"-force", "-verbose"}; "-force", "-verbose"};
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
removeBaseAndBucketOverrides(conf,
S3_METADATA_STORE_IMPL, S3A_BUCKET_PROBE);
conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_LOCAL);
conf.setInt(S3A_BUCKET_PROBE, S3A_BUCKET_PROBE_DEFAULT);
return conf;
}
@Override @Override
public void setup() throws Exception { public void setup() throws Exception {
super.setup(); super.setup();
MetadataStore ms = getMetadataStore(); assertTrue("metadata store impl should be LocalMetadataStore.",
Assume.assumeTrue("Test only applies when a local store is used for S3Guard;" getMetadataStore() instanceof LocalMetadataStore);
+ "Store is " + (ms == null ? "none" : ms.toString()),
ms instanceof LocalMetadataStore);
} }
@Test @Test
@ -97,7 +110,6 @@ public void testImportCommand() throws Exception {
.getListing().size()); .getListing().size());
assertEquals("Expected 2 items: empty directory and a parent directory", 2, assertEquals("Expected 2 items: empty directory and a parent directory", 2,
ms.listChildren(parent).getListing().size()); ms.listChildren(parent).getListing().size());
assertTrue(children.isAuthoritative());
} }
@Test @Test
@ -165,20 +177,12 @@ public void testImportNoFilesystem() throws Throwable {
@Test @Test
public void testInfoBucketAndRegionNoFS() throws Throwable { public void testInfoBucketAndRegionNoFS() throws Throwable {
intercept(FileNotFoundException.class, intercept(UnknownStoreException.class,
() -> run(BucketInfo.NAME, "-meta", () -> run(BucketInfo.NAME, "-meta",
LOCAL_METADATA, "-region", LOCAL_METADATA, "-region",
"any-region", S3A_THIS_BUCKET_DOES_NOT_EXIST)); "any-region", S3A_THIS_BUCKET_DOES_NOT_EXIST));
} }
@Test
public void testInitNegativeRead() throws Throwable {
runToFailure(INVALID_ARGUMENT,
Init.NAME, "-meta", LOCAL_METADATA, "-region",
"eu-west-1",
READ_FLAG, "-10");
}
@Test @Test
public void testInit() throws Throwable { public void testInit() throws Throwable {
run(Init.NAME, run(Init.NAME,