big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15635. s3guard set-capacity command to fail fast if bucket is unguarded.

Browse Source 
Contributed by Gabor Bota.
This commit is contained in:
Sean Mackrory 2018-09-12 08:07:10 -06:00
parent 47299ce23f
commit 47b72c87eb
2 changed files with 32 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
View File

@ -479,6 +479,20 @@ public String getUsage() {
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
Map<String, String> options = new HashMap<>();
String s3Path = paths.get(0);
// Check if DynamoDB url is set from arguments.
String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
if(metadataStoreUri == null || metadataStoreUri.isEmpty()) {
// If not set, check if filesystem is guarded by creating an
// S3AFileSystem and check if hasMetadataStore is true
try (S3AFileSystem s3AFileSystem = (S3AFileSystem)
S3AFileSystem.newInstance(toUri(s3Path), getConf())){
Preconditions.checkState(s3AFileSystem.hasMetadataStore(),
"The S3 bucket is unguarded. " + getName()
+ " can not be used on an unguarded bucket.");
}
}
String readCap = getCommandFormat().getOptValue(READ_FLAG);
if (StringUtils.isNotEmpty(readCap)) {

19
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
View File

@ -29,6 +29,7 @@
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
@ -53,6 +54,7 @@
import org.apache.hadoop.util.StringUtils;
import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CREATE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
@ -296,7 +298,7 @@ public void testPruneCommandConf() throws Exception {
}
@Test
public void testSetCapacityFailFast() throws Exception{
public void testSetCapacityFailFastOnReadWriteOfZero() throws Exception{
Configuration conf = getConfiguration();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, getFileSystem().getBucket());
@ -312,6 +314,21 @@ public void testSetCapacityFailFast() throws Exception{
S3GuardTool.SetCapacity.WRITE_CAP_INVALID, () -> cmdW.run(argsW));
}
@Test
public void testSetCapacityFailFastIfNotGuarded() throws Exception{
Configuration conf = getConfiguration();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString());
conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, Boolean.FALSE.toString());
conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
S3GuardTool.SetCapacity cmdR = new S3GuardTool.SetCapacity(conf);
String[] argsR = new String[]{cmdR.getName(),
"s3a://" + getFileSystem().getBucket()};
intercept(IllegalStateException.class, "unguarded",
() -> run(argsR));
}
@Test
public void testDestroyNoBucket() throws Throwable {
intercept(FileNotFoundException.class,