diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md index a5aaae9145..649c80a22d 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md @@ -934,6 +934,21 @@ Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: GZIP is not applic ... ``` + +### AWSBadRequestException `UnsupportedStorageClass` + +S3 Select doesn't work with some storage classes like Glacier or Reduced Redundancy. +Make sure you've set `fs.s3a.create.storage.class` to a supported storage class for S3 Select. + +``` +org.apache.hadoop.fs.s3a.AWSBadRequestException: + Select on s3a://example/dataset.csv.gz: + com.amazonaws.services.s3.model.AmazonS3Exception: + We do not support REDUCED_REDUNDANCY storage class. + Please check the service documentation and try again. + (Service: Amazon S3; Status Code: 400; Error Code: UnsupportedStorageClass +``` + ### `PathIOException`: "seek() not supported" The input stream returned by the select call does not support seeking diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java index bf5d96e73b..2c1a10a21d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java @@ -60,7 +60,9 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.hadoop.util.DurationInfo; +import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getLandsatCSVPath; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.select.CsvFile.ALL_QUOTES; import static org.apache.hadoop.fs.s3a.select.SelectConstants.*; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -280,6 +282,14 @@ boolean isSelectAvailable(final FileSystem filesystem) { .hasCapability(S3_SELECT_CAPABILITY); } + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + removeBaseAndBucketOverrides(conf, STORAGE_CLASS); + + return conf; + } + /** * Setup: requires select to be available. */