diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java index 0a1480adff..db94bbbb75 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java @@ -17,39 +17,60 @@ */ package org.apache.hadoop.ozone.s3; +import javax.annotation.Priority; import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.container.ContainerRequestFilter; import javax.ws.rs.container.PreMatching; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.ext.Provider; import java.io.IOException; /** * Filter to adjust request headers for compatible reasons. + * + * It should be executed AFTER signature check (VirtualHostStyleFilter) as the + * original Content-Type could be part of the base of the signature. */ - @Provider @PreMatching +@Priority(VirtualHostStyleFilter.PRIORITY + + S3GatewayHttpServer.FILTER_PRIORITY_DO_AFTER) public class HeaderPreprocessor implements ContainerRequestFilter { + public static final String MULTIPART_UPLOAD_MARKER = "ozone/mpu"; + @Override public void filter(ContainerRequestContext requestContext) throws IOException { - if (requestContext.getUriInfo().getQueryParameters() - .containsKey("delete")) { + MultivaluedMap queryParameters = + requestContext.getUriInfo().getQueryParameters(); + + if (queryParameters.containsKey("delete")) { //aws cli doesn't send proper Content-Type and by default POST requests //processed as form-url-encoded. Here we can fix this. requestContext.getHeaders() .putSingle("Content-Type", MediaType.APPLICATION_XML); } - if (requestContext.getUriInfo().getQueryParameters() - .containsKey("uploadId")) { + if (queryParameters.containsKey("uploadId")) { //aws cli doesn't send proper Content-Type and by default POST requests //processed as form-url-encoded. Here we can fix this. requestContext.getHeaders() .putSingle("Content-Type", MediaType.APPLICATION_XML); + } else if (queryParameters.containsKey("uploads")) { + // uploads defined but uploadId is not --> this is the creation of the + // multi-part-upload requests. + // + //In AWS SDK for go uses application/octet-stream which also + //should be fixed to route the request to the right jaxrs method. + // + //Should be empty instead of XML as the body is empty which can not be + //serialized as as CompleteMultipartUploadRequest + requestContext.getHeaders() + .putSingle("Content-Type", MULTIPART_UPLOAD_MARKER); } + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java index f20b928b1f..f3d83412ae 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java @@ -27,6 +27,11 @@ import org.apache.hadoop.hdds.server.BaseHttpServer; */ public class S3GatewayHttpServer extends BaseHttpServer { + /** + * Default offset between two filters. + */ + public static final int FILTER_PRIORITY_DO_AFTER = 50; + public S3GatewayHttpServer(Configuration conf, String name) throws IOException { super(conf, name); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java index 50014fea92..9ce98e11ee 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.s3; +import javax.annotation.Priority; import javax.inject.Inject; import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.container.ContainerRequestFilter; @@ -46,8 +47,11 @@ import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NA @Provider @PreMatching +@Priority(VirtualHostStyleFilter.PRIORITY) public class VirtualHostStyleFilter implements ContainerRequestFilter { + public static final int PRIORITY = 100; + private static final Logger LOG = LoggerFactory.getLogger( VirtualHostStyleFilter.class); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index b520e7be0b..70bfb7f8e0 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.s3.endpoint; +import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -58,6 +59,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.SignedChunksInputStream; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; @@ -417,33 +419,19 @@ public class ObjectEndpoint extends EndpointBase { } + /** + * Initialize MultiPartUpload request. + *

+ * Note: the specific content type is set by the HeaderPreprocessor. + */ @POST @Produces(MediaType.APPLICATION_XML) - public Response multipartUpload( + @Consumes(HeaderPreprocessor.MULTIPART_UPLOAD_MARKER) + public Response initializeMultipartUpload( @PathParam("bucket") String bucket, - @PathParam("path") String key, - @QueryParam("uploads") String uploads, - @QueryParam("uploadId") @DefaultValue("") String uploadID, - CompleteMultipartUploadRequest request) throws IOException, OS3Exception { - if (!uploadID.equals("")) { - //Complete Multipart upload request. - return completeMultipartUpload(bucket, key, uploadID, request); - } else { - // Initiate Multipart upload request. - return initiateMultipartUpload(bucket, key); - } - } - - /** - * Initiate Multipart upload request. - * @param bucket - * @param key - * @return Response - * @throws IOException - * @throws OS3Exception - */ - private Response initiateMultipartUpload(String bucket, String key) throws - IOException, OS3Exception { + @PathParam("path") String key + ) + throws IOException, OS3Exception { try { OzoneBucket ozoneBucket = getBucket(bucket); String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER); @@ -473,7 +461,6 @@ public class ObjectEndpoint extends EndpointBase { multipartUploadInitiateResponse.setKey(key); multipartUploadInitiateResponse.setUploadID(multipartInfo.getUploadID()); - return Response.status(Status.OK).entity( multipartUploadInitiateResponse).build(); } catch (IOException ex) { @@ -484,18 +471,15 @@ public class ObjectEndpoint extends EndpointBase { } /** - * Complete Multipart upload request. - * @param bucket - * @param key - * @param uploadID - * @param multipartUploadRequest - * @return Response - * @throws IOException - * @throws OS3Exception + * Complete a multipart upload. */ - private Response completeMultipartUpload(String bucket, String key, String - uploadID, CompleteMultipartUploadRequest multipartUploadRequest) throws - IOException, OS3Exception { + @POST + @Produces(MediaType.APPLICATION_XML) + public Response completeMultipartUpload(@PathParam("bucket") String bucket, + @PathParam("path") String key, + @QueryParam("uploadId") @DefaultValue("") String uploadID, + CompleteMultipartUploadRequest multipartUploadRequest) + throws IOException, OS3Exception { OzoneBucket ozoneBucket = getBucket(bucket); Map partsMap = new TreeMap<>(); List partList = diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java index 76d4a12b7b..912a769cd3 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java @@ -56,7 +56,7 @@ public class TestAbortMultipartUpload { rest.setHeaders(headers); rest.setClient(client); - Response response = rest.multipartUpload(bucket, key, "", "", null); + Response response = rest.initializeMultipartUpload(bucket, key); assertEquals(response.getStatus(), 200); MultipartUploadInitiateResponse multipartUploadInitiateResponse = diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java index 6f48ecb509..212721af00 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java @@ -60,7 +60,7 @@ public class TestInitiateMultipartUpload { rest.setHeaders(headers); rest.setClient(client); - Response response = rest.multipartUpload(bucket, key, "", "", null); + Response response = rest.initializeMultipartUpload(bucket, key); assertEquals(response.getStatus(), 200); MultipartUploadInitiateResponse multipartUploadInitiateResponse = @@ -69,7 +69,7 @@ public class TestInitiateMultipartUpload { String uploadID = multipartUploadInitiateResponse.getUploadID(); // Calling again should return different uploadID. - response = rest.multipartUpload(bucket, key, "", "", null); + response = rest.initializeMultipartUpload(bucket, key); assertEquals(response.getStatus(), 200); multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java index ac6aa72e4f..21545ec9b0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java @@ -61,7 +61,7 @@ public class TestListParts { REST.setHeaders(headers); REST.setClient(client); - Response response = REST.multipartUpload(BUCKET, KEY, "", "", null); + Response response = REST.initializeMultipartUpload(BUCKET, KEY); MultipartUploadInitiateResponse multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); assertNotNull(multipartUploadInitiateResponse.getUploadID()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java index 8be61310f9..b9e3885ac6 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java @@ -70,7 +70,7 @@ public class TestMultipartUploadComplete { private String initiateMultipartUpload(String key) throws IOException, OS3Exception { - Response response = REST.multipartUpload(BUCKET, key, "", "", null); + Response response = REST.initializeMultipartUpload(BUCKET, key); MultipartUploadInitiateResponse multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); assertNotNull(multipartUploadInitiateResponse.getUploadID()); @@ -99,7 +99,7 @@ public class TestMultipartUploadComplete { private void completeMultipartUpload(String key, CompleteMultipartUploadRequest completeMultipartUploadRequest, String uploadID) throws IOException, OS3Exception { - Response response = REST.multipartUpload(BUCKET, key, "", uploadID, + Response response = REST.completeMultipartUpload(BUCKET, key, uploadID, completeMultipartUploadRequest); assertEquals(response.getStatus(), 200); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java index 120fbb2f2e..3e91a77ffd 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java @@ -67,7 +67,7 @@ public class TestPartUpload { @Test public void testPartUpload() throws Exception { - Response response = REST.multipartUpload(BUCKET, KEY, "", "", null); + Response response = REST.initializeMultipartUpload(BUCKET, KEY); MultipartUploadInitiateResponse multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); assertNotNull(multipartUploadInitiateResponse.getUploadID()); @@ -86,7 +86,7 @@ public class TestPartUpload { @Test public void testPartUploadWithOverride() throws Exception { - Response response = REST.multipartUpload(BUCKET, KEY, "", "", null); + Response response = REST.initializeMultipartUpload(BUCKET, KEY); MultipartUploadInitiateResponse multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); assertNotNull(multipartUploadInitiateResponse.getUploadID());