big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-1948. S3 MPU can't be created with octet-stream content-type (#1266)

Browse Source
This commit is contained in:
Elek, Márton 2019-08-23 08:15:43 +02:00 committed by Bharat Viswanadham
parent abc8fde4ca
commit edd708527d
9 changed files with 63 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java
View File

@ -17,39 +17,60 @@
*/
package org.apache.hadoop.ozone.s3;
import javax.annotation.Priority;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import javax.ws.rs.container.PreMatching;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.ext.Provider;
import java.io.IOException;
/**
* Filter to adjust request headers for compatible reasons.
*
* It should be executed AFTER signature check (VirtualHostStyleFilter) as the
* original Content-Type could be part of the base of the signature.
*/
@Provider
@PreMatching
@Priority(VirtualHostStyleFilter.PRIORITY
+ S3GatewayHttpServer.FILTER_PRIORITY_DO_AFTER)
public class HeaderPreprocessor implements ContainerRequestFilter {
public static final String MULTIPART_UPLOAD_MARKER = "ozone/mpu";
@Override
public void filter(ContainerRequestContext requestContext) throws
IOException {
if (requestContext.getUriInfo().getQueryParameters()
.containsKey("delete")) {
MultivaluedMap<String, String> queryParameters =
requestContext.getUriInfo().getQueryParameters();
if (queryParameters.containsKey("delete")) {
//aws cli doesn't send proper Content-Type and by default POST requests
//processed as form-url-encoded. Here we can fix this.
requestContext.getHeaders()
.putSingle("Content-Type", MediaType.APPLICATION_XML);
}
if (requestContext.getUriInfo().getQueryParameters()
.containsKey("uploadId")) {
if (queryParameters.containsKey("uploadId")) {
//aws cli doesn't send proper Content-Type and by default POST requests
//processed as form-url-encoded. Here we can fix this.
requestContext.getHeaders()
.putSingle("Content-Type", MediaType.APPLICATION_XML);
} else if (queryParameters.containsKey("uploads")) {
// uploads defined but uploadId is not --> this is the creation of the
// multi-part-upload requests.
//
//In AWS SDK for go uses application/octet-stream which also
//should be fixed to route the request to the right jaxrs method.
//
//Should be empty instead of XML as the body is empty which can not be
//serialized as as CompleteMultipartUploadRequest
requestContext.getHeaders()
.putSingle("Content-Type", MULTIPART_UPLOAD_MARKER);
}
}
}

5
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java
View File

@ -27,6 +27,11 @@
*/
public class S3GatewayHttpServer extends BaseHttpServer {
/**
* Default offset between two filters.
*/
public static final int FILTER_PRIORITY_DO_AFTER = 50;
public S3GatewayHttpServer(Configuration conf,
String name) throws IOException {
super(conf, name);

4
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.s3;
import javax.annotation.Priority;
import javax.inject.Inject;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
@ -46,8 +47,11 @@
@Provider
@PreMatching
@Priority(VirtualHostStyleFilter.PRIORITY)
public class VirtualHostStyleFilter implements ContainerRequestFilter {
public static final int PRIORITY = 100;
private static final Logger LOG = LoggerFactory.getLogger(
VirtualHostStyleFilter.class);

56
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.s3.endpoint;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
@ -58,6 +59,7 @@
import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.s3.HeaderPreprocessor;
import org.apache.hadoop.ozone.s3.SignedChunksInputStream;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
@ -417,33 +419,19 @@ public Response delete(
}
/**
* Initialize MultiPartUpload request.
* <p>
* Note: the specific content type is set by the HeaderPreprocessor.
*/
@POST
@Produces(MediaType.APPLICATION_XML)
public Response multipartUpload(
@Consumes(HeaderPreprocessor.MULTIPART_UPLOAD_MARKER)
public Response initializeMultipartUpload(
@PathParam("bucket") String bucket,
@PathParam("path") String key,
@QueryParam("uploads") String uploads,
@QueryParam("uploadId") @DefaultValue("") String uploadID,
CompleteMultipartUploadRequest request) throws IOException, OS3Exception {
if (!uploadID.equals("")) {
//Complete Multipart upload request.
return completeMultipartUpload(bucket, key, uploadID, request);
} else {
// Initiate Multipart upload request.
return initiateMultipartUpload(bucket, key);
}
}
/**
* Initiate Multipart upload request.
* @param bucket
* @param key
* @return Response
* @throws IOException
* @throws OS3Exception
*/
private Response initiateMultipartUpload(String bucket, String key) throws
IOException, OS3Exception {
@PathParam("path") String key
)
throws IOException, OS3Exception {
try {
OzoneBucket ozoneBucket = getBucket(bucket);
String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER);
@ -473,7 +461,6 @@ private Response initiateMultipartUpload(String bucket, String key) throws
multipartUploadInitiateResponse.setKey(key);
multipartUploadInitiateResponse.setUploadID(multipartInfo.getUploadID());
return Response.status(Status.OK).entity(
multipartUploadInitiateResponse).build();
} catch (IOException ex) {
@ -484,18 +471,15 @@ private Response initiateMultipartUpload(String bucket, String key) throws
}
/**
* Complete Multipart upload request.
* @param bucket
* @param key
* @param uploadID
* @param multipartUploadRequest
* @return Response
* @throws IOException
* @throws OS3Exception
* Complete a multipart upload.
*/
private Response completeMultipartUpload(String bucket, String key, String
uploadID, CompleteMultipartUploadRequest multipartUploadRequest) throws
IOException, OS3Exception {
@POST
@Produces(MediaType.APPLICATION_XML)
public Response completeMultipartUpload(@PathParam("bucket") String bucket,
@PathParam("path") String key,
@QueryParam("uploadId") @DefaultValue("") String uploadID,
CompleteMultipartUploadRequest multipartUploadRequest)
throws IOException, OS3Exception {
OzoneBucket ozoneBucket = getBucket(bucket);
Map<Integer, String> partsMap = new TreeMap<>();
List<CompleteMultipartUploadRequest.Part> partList =

2
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
View File

@ -56,7 +56,7 @@ public void testAbortMultipartUpload() throws Exception {
rest.setHeaders(headers);
rest.setClient(client);
Response response = rest.multipartUpload(bucket, key, "", "", null);
Response response = rest.initializeMultipartUpload(bucket, key);
assertEquals(response.getStatus(), 200);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =

4
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
View File

@ -60,7 +60,7 @@ public void testInitiateMultipartUpload() throws Exception {
rest.setHeaders(headers);
rest.setClient(client);
Response response = rest.multipartUpload(bucket, key, "", "", null);
Response response = rest.initializeMultipartUpload(bucket, key);
assertEquals(response.getStatus(), 200);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
@ -69,7 +69,7 @@ public void testInitiateMultipartUpload() throws Exception {
String uploadID = multipartUploadInitiateResponse.getUploadID();
// Calling again should return different uploadID.
response = rest.multipartUpload(bucket, key, "", "", null);
response = rest.initializeMultipartUpload(bucket, key);
assertEquals(response.getStatus(), 200);
multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();

2
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
View File

@ -61,7 +61,7 @@ public static void setUp() throws Exception {
REST.setHeaders(headers);
REST.setClient(client);
Response response = REST.multipartUpload(BUCKET, KEY, "", "", null);
Response response = REST.initializeMultipartUpload(BUCKET, KEY);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());

4
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
View File

@ -70,7 +70,7 @@ public static void setUp() throws Exception {
private String initiateMultipartUpload(String key) throws IOException,
OS3Exception {
Response response = REST.multipartUpload(BUCKET, key, "", "", null);
Response response = REST.initializeMultipartUpload(BUCKET, key);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
@ -99,7 +99,7 @@ private Part uploadPart(String key, String uploadID, int partNumber, String
private void completeMultipartUpload(String key,
CompleteMultipartUploadRequest completeMultipartUploadRequest,
String uploadID) throws IOException, OS3Exception {
Response response = REST.multipartUpload(BUCKET, key, "", uploadID,
Response response = REST.completeMultipartUpload(BUCKET, key, uploadID,
completeMultipartUploadRequest);
assertEquals(response.getStatus(), 200);

4
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
View File

@ -67,7 +67,7 @@ public static void setUp() throws Exception {
@Test
public void testPartUpload() throws Exception {
Response response = REST.multipartUpload(BUCKET, KEY, "", "", null);
Response response = REST.initializeMultipartUpload(BUCKET, KEY);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
@ -86,7 +86,7 @@ public void testPartUpload() throws Exception {
@Test
public void testPartUploadWithOverride() throws Exception {
Response response = REST.multipartUpload(BUCKET, KEY, "", "", null);
Response response = REST.initializeMultipartUpload(BUCKET, KEY);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());