big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15843. s3guard bucket-info command to not print a stack trace on bucket-not-found.

Browse Source 
Contributed by Adam Antal.
This commit is contained in:
Adam Antal 2019-01-14 17:27:00 +00:00 committed by Steve Loughran
parent 04fcbef9c9
commit c4a00d1ad3
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
3 changed files with 128 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
View File

@ -66,6 +66,7 @@
* CLI to manage S3Guard Metadata Store.
*/
public abstract class S3GuardTool extends Configured implements Tool {
private static final Logger LOG = LoggerFactory.getLogger(S3GuardTool.class);
private static final String NAME = "s3guard";
@ -91,6 +92,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
private static final String DATA_IN_S3_IS_PRESERVED
= "(all data in S3 is preserved)";
public static final String E_NO_METASTORE_OR_FILESYSTEM
= "No metastore or filesystem specified";
abstract public String getUsage();
// Exit codes
@ -267,12 +271,8 @@ MetadataStore initMetadataStore(boolean forceCreate) throws IOException {
if (getStore() != null) {
return getStore();
}
Configuration conf;
if (filesystem == null) {
conf = getConf();
} else {
conf = filesystem.getConf();
}
final boolean hasFileSystem = filesystem != null;
final Configuration conf = hasFileSystem ? filesystem.getConf() : getConf();
String metaURI = getCommandFormat().getOptValue(META_FLAG);
if (metaURI != null && !metaURI.isEmpty()) {
URI uri = URI.create(metaURI);
@ -294,6 +294,13 @@ MetadataStore initMetadataStore(boolean forceCreate) throws IOException {
String.format("Metadata store %s is not supported", uri));
}
} else {
if (!hasFileSystem) {
// command didn't declare a metadata store URI or a bucket.
// to avoid problems related to picking up a shared table for actions
// line init and destroy (HADOOP-15843), this is rejected
printHelp(this);
throw usageError(E_NO_METASTORE_OR_FILESYSTEM);
}
// CLI does not specify metadata store URI, it uses default metadata store
// DynamoDB instead.
setStore(new DynamoDBMetadataStore());
@ -302,10 +309,10 @@ MetadataStore initMetadataStore(boolean forceCreate) throws IOException {
}
}
if (filesystem == null) {
getStore().initialize(conf);
} else {
if (hasFileSystem) {
getStore().initialize(filesystem);
} else {
getStore().initialize(conf);
}
LOG.info("Metadata store {} is initialized.", getStore());
return getStore();
@ -1431,13 +1438,13 @@ protected static URI toUri(String s3Path) {
return uri;
}
private static void printHelp() {
if (command == null) {
private static void printHelp(S3GuardTool tool) {
if (tool == null) {
errorln("Usage: hadoop " + USAGE);
errorln("\tperform S3Guard metadata store " +
"administrative commands.");
} else {
errorln("Usage: hadoop " + command.getUsage());
errorln("Usage: hadoop " + tool.getUsage());
}
errorln();
errorln(COMMON_USAGE);
@ -1477,7 +1484,6 @@ protected static void printStoreDiagnostics(PrintStream out,
}
}
/**
* Handle store not found by converting to an exit exception
* with specific error code.
@ -1525,6 +1531,18 @@ protected static ExitUtil.ExitException userAborted(
return new ExitUtil.ExitException(ERROR, String.format(format, args));
}
/**
* Build the exception to raise on a usage error
* @param format string format
* @param args optional arguments for the string
* @return a new exception to throw
*/
protected static ExitUtil.ExitException usageError(
String format, Object...args) {
return new ExitUtil.ExitException(E_USAGE, String.format(format, args));
}
/**
* Execute the command with the given arguments.
*
@ -1540,8 +1558,8 @@ public static int run(Configuration conf, String...args) throws
String[] otherArgs = new GenericOptionsParser(conf, args)
.getRemainingArgs();
if (otherArgs.length == 0) {
printHelp();
throw new ExitUtil.ExitException(E_USAGE, "No arguments provided");
printHelp(null);
throw usageError("No arguments provided");
}
final String subCommand = otherArgs[0];
LOG.debug("Executing command {}", subCommand);
@ -1571,7 +1589,7 @@ public static int run(Configuration conf, String...args) throws
command = new Uploads(conf);
break;
default:
printHelp();
printHelp(null);
throw new ExitUtil.ExitException(E_USAGE,
"Unknown command " + subCommand);
}
@ -1588,11 +1606,17 @@ public static void main(String[] args) {
exit(ret, "");
} catch (CommandFormat.UnknownOptionException e) {
errorln(e.getMessage());
printHelp();
printHelp(command);
exit(E_USAGE, e.getMessage());
} catch (ExitUtil.ExitException e) {
// explicitly raised exit code
exit(e.getExitCode(), e.toString());
} catch (FileNotFoundException e) {
// bucket doesn't exist or similar.
// skip the stack trace and choose the return code of 44, "404"
errorln(e.toString());
LOG.debug("Not found:", e);
exit(EXIT_NOT_FOUND, e.toString());
} catch (Throwable e) {
e.printStackTrace(System.err);
exit(ERROR, e.toString());

14
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
View File

@ -1006,6 +1006,20 @@ There's are limit on how often you can change the capacity of an DynamoDB table;
if you call set-capacity too often, it fails. Wait until the after the time indicated
and try again.
### Error `Invalid region specified`
```
java.io.IOException: Invalid region specified "iceland-2":
Region can be configured with fs.s3a.s3guard.ddb.region:
us-gov-west-1, us-east-1, us-east-2, us-west-1, us-west-2,
eu-west-1, eu-west-2, eu-west-3, eu-central-1, ap-south-1,
ap-southeast-1, ap-southeast-2, ap-northeast-1, ap-northeast-2,
sa-east-1, cn-north-1, cn-northwest-1, ca-central-1
at org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory$DefaultDynamoDBClientFactory.getRegion
at org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory$DefaultDynamoDBClientFactory.createDynamoDBClient
```
The region specified in `fs.s3a.s3guard.ddb.region` is invalid.
## Other Topics

86
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
View File

@ -54,12 +54,15 @@
import org.apache.hadoop.util.StringUtils;
import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CREATE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_NO_METASTORE_OR_FILESYSTEM;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_USAGE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@ -337,28 +340,68 @@ public void testBucketInfoUnguarded() throws Exception {
@Test
public void testSetCapacityFailFastIfNotGuarded() throws Exception{
Configuration conf = getConfiguration();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString());
conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, Boolean.FALSE.toString());
bindToNonexistentTable(conf);
String bucket = rawFs.getBucket();
clearBucketOption(conf, bucket, S3_METADATA_STORE_IMPL);
clearBucketOption(conf, bucket, S3GUARD_DDB_TABLE_NAME_KEY);
clearBucketOption(conf, bucket, S3GUARD_DDB_TABLE_CREATE_KEY);
conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
S3GuardTool.SetCapacity cmdR = new S3GuardTool.SetCapacity(conf);
String[] argsR = new String[]{cmdR.getName(),
"s3a://" + getFileSystem().getBucket()};
String[] argsR = new String[]{
cmdR.getName(),
"s3a://" + getFileSystem().getBucket()
};
intercept(IllegalStateException.class, "unguarded",
() -> run(argsR));
() -> cmdR.run(argsR));
}
/**
* Binds the configuration to a nonexistent table.
* @param conf
*/
protected void bindToNonexistentTable(final Configuration conf) {
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString());
conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false);
}
@Test
public void testDestroyNoBucket() throws Throwable {
describe("Destroy a bucket which doesn't exist");
Configuration conf = getConfiguration();
// set a table as a safety check in case the test goes wrong
// and deletes it.
bindToNonexistentTable(conf);
S3GuardTool.Destroy cmdR = new S3GuardTool.Destroy(conf);
String[] argsR = new String[]{
S3GuardTool.Destroy.NAME,
S3A_THIS_BUCKET_DOES_NOT_EXIST
};
intercept(FileNotFoundException.class,
new Callable<Integer>() {
@Override
public Integer call() throws Exception {
return run(S3GuardTool.Destroy.NAME,
S3A_THIS_BUCKET_DOES_NOT_EXIST);
() -> cmdR.run(argsR));
}
});
@Test
public void testDestroyNoArgs() throws Throwable {
describe("Destroy a bucket which doesn't exist");
Configuration conf = getConfiguration();
// set a table as a safety check in case the test goes wrong
// and deletes it.
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString());
conf.set(S3GUARD_DDB_REGION_KEY, "us-gov-west-1");
conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false);
S3GuardTool.Destroy cmdR = new S3GuardTool.Destroy(conf);
assertExitCode(E_USAGE,
intercept(ExitUtil.ExitException.class,
E_NO_METASTORE_OR_FILESYSTEM,
() -> cmdR.run(new String[]{})));
}
@Test
@ -373,12 +416,25 @@ public void testProbeForMagic() throws Throwable {
exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name);
} else {
// if the FS isn't magic, expect the probe to fail
ExitUtil.ExitException e = intercept(ExitUtil.ExitException.class,
() -> exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name));
if (e.getExitCode() != E_BAD_STATE) {
throw e;
assertExitCode(E_BAD_STATE,
intercept(ExitUtil.ExitException.class,
() -> exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name)));
}
}
/**
* Assert that an exit exception had a specific error code.
* @param expectedErrorCode expected code.
* @param e exit exception
* @throws AssertionError with the exit exception nested inside
*/
protected void assertExitCode(final int expectedErrorCode,
final ExitUtil.ExitException e) {
if (e.getExitCode() != expectedErrorCode) {
throw new AssertionError("Expected error code " + expectedErrorCode
+ " in " + e,
e);
}
}
@Test