diff --git a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot index eb3a8bb547..7f8cd4b6b6 100644 --- a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot @@ -28,4 +28,10 @@ Execute and checkrc ${rc} ${output} = Run And Return Rc And Output ${command} Log ${output} Should Be Equal As Integers ${rc} ${expected_error_code} - [return] ${output} \ No newline at end of file + [return] ${output} + +Compare files + [arguments] ${file1} ${file2} + ${checksumbefore} = Execute md5sum ${file1} | awk '{print $1}' + ${checksumafter} = Execute md5sum ${file2} | awk '{print $1}' + Should Be Equal ${checksumbefore} ${checksumafter} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index 1f69c9e93f..0133d50ff1 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -22,11 +22,9 @@ Resource commonawslib.robot Test Setup Setup s3 tests *** Keywords *** -Create Random file for mac - Execute dd if=/dev/urandom of=/tmp/part1 bs=1m count=5 - -Create Random file for linux - Execute dd if=/dev/urandom of=/tmp/part1 bs=1M count=5 +Create Random file + [arguments] ${size_in_megabytes} + Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576 count=${size_in_megabytes} *** Variables *** @@ -54,16 +52,13 @@ Test Multipart Upload # upload we get error entity too small. So, considering further complete # multipart upload, uploading each part as 5MB file, exception is for last part - ${system} = Evaluate platform.system() platform - Run Keyword if '${system}' == 'Darwin' Create Random file for mac - Run Keyword if '${system}' == 'Linux' Create Random file for linux - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} - Should contain ${result} ETag + Run Keyword Create Random file 5 + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} + Should contain ${result} ETag # override part - Run Keyword if '${system}' == 'Darwin' Create Random file for mac - Run Keyword if '${system}' == 'Linux' Create Random file for linux - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} - Should contain ${result} ETag + Run Keyword Create Random file 5 + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} + Should contain ${result} ETag Test Multipart Upload Complete @@ -74,17 +69,15 @@ Test Multipart Upload Complete Should contain ${result} UploadId #upload parts - ${system} = Evaluate platform.system() platform - Run Keyword if '${system}' == 'Darwin' Create Random file for mac - Run Keyword if '${system}' == 'Linux' Create Random file for linux - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} - ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + Run Keyword Create Random file 5 + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} + ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 + Should contain ${result} ETag Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} - ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} + ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 + Should contain ${result} ETag #complete multipart upload ${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' @@ -94,11 +87,8 @@ Test Multipart Upload Complete #read file and check the key ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key multipartKey1 /tmp/multipartKey1.result - Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartkey1 - ${checksumbefore} = Execute md5sum /tmp/multipartkey1 | awk '{print $1}' - ${checksumafter} = Execute md5sum /tmp/multipartKey1.result | awk '{print $1}' - Should Be Equal ${checksumbefore} ${checksumafter} - + Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartKey1 + Compare files /tmp/multipartKey1 /tmp/multipartKey1.result Test Multipart Upload Complete Entity too small ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey2 @@ -109,14 +99,14 @@ Test Multipart Upload Complete Entity too small #upload parts Execute echo "Part1" > /tmp/part1 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} - ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} + ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 + Should contain ${result} ETag Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} - ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} + ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 + Should contain ${result} ETag #complete multipart upload ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255 @@ -132,14 +122,14 @@ Test Multipart Upload Complete Invalid part #upload parts Execute echo "Part1" > /tmp/part1 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} - ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} + ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 + Should contain ${result} ETag Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} - ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} + ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 + Should contain ${result} ETag #complete multipart upload ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 @@ -158,9 +148,9 @@ Test abort Multipart upload with invalid uploadId ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id "random" 255 Upload part with Incorrect uploadID - Execute echo "Multipart upload" > /tmp/testfile - ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255 - Should contain ${result} NoSuchUpload + Execute echo "Multipart upload" > /tmp/testfile + ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255 + Should contain ${result} NoSuchUpload Test list parts #initiate multipart upload @@ -171,37 +161,42 @@ Test list parts Should contain ${result} UploadId #upload parts - ${system} = Evaluate platform.system() platform - Run Keyword if '${system}' == 'Darwin' Create Random file for mac - Run Keyword if '${system}' == 'Linux' Create Random file for linux - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} - ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + Run Keyword Create Random file 5 + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} + ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 + Should contain ${result} ETag - Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} - ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + Execute echo "Part2" > /tmp/part2 + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} + ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 + Should contain ${result} ETag #list parts - ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} - ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 - ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0 - Should Be equal ${part1} ${eTag1} - Should contain ${part2} ${eTag2} - Should contain ${result} STANDARD + ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} + ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 + ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0 + Should Be equal ${part1} ${eTag1} + Should contain ${part2} ${eTag2} + Should contain ${result} STANDARD #list parts with max-items and next token - ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 - ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 - ${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0 - Should Be equal ${part1} ${eTag1} - Should contain ${result} STANDARD + ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 + ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 + ${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0 + Should Be equal ${part1} ${eTag1} + Should contain ${result} STANDARD - ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token} - ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 - Should Be equal ${part2} ${eTag2} - Should contain ${result} STANDARD + ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token} + ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 + Should Be equal ${part2} ${eTag2} + Should contain ${result} STANDARD #finally abort it - ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0 + ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0 + +Test Multipart Upload with the simplified aws s3 cp API + Create Random file 22 + Execute AWSS3Cli cp /tmp/part1 s3://${BUCKET}/mpyawscli + Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli /tmp/part1.result + Execute AWSS3Cli rm s3://${BUCKET}/mpyawscli + Compare files /tmp/part1 /tmp/part1.result diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index e42ffaf938..1b2a5048ef 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -42,9 +42,7 @@ Put object to s3 #This test depends on the previous test case. Can't be executes alone Get object from s3 ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result - ${checksumbefore} = Execute md5sum /tmp/testfile | awk '{print $1}' - ${checksumafter} = Execute md5sum /tmp/testfile.result | awk '{print $1}' - Should Be Equal ${checksumbefore} ${checksumafter} + Compare files /tmp/testfile /tmp/testfile.result Get Partial object from s3 with both start and endoffset ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PlainTextMultipartUploadReader.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PlainTextMultipartUploadReader.java new file mode 100644 index 0000000000..599b473194 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PlainTextMultipartUploadReader.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.endpoint; + +import javax.ws.rs.Consumes; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.MessageBodyReader; +import javax.ws.rs.ext.Provider; +import java.io.IOException; +import java.io.InputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; + +/** + * Body reader to accept plain text MPU. + *

+ * Aws s3 api sends a multipartupload request with the content type + * 'text/plain' in case of using 'aws s3 cp' (instead of aws s3api). + *

+ * Our generic ObjectEndpoint.multipartUpload has a + * CompleteMultipartUploadRequest parameter, which is required only for the + * completion request. + *

+ * But JaxRS tries to parse it from the body for the requests and in case of + * text/plain requests this parsing is failed. This simple BodyReader enables + * to parse an empty text/plain message and return with an empty completion + * request. + */ +@Provider +@Consumes("text/plain") +public class PlainTextMultipartUploadReader + implements MessageBodyReader { + + @Override + public boolean isReadable(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return type.equals(CompleteMultipartUploadRequest.class) + && mediaType.equals(MediaType.TEXT_PLAIN_TYPE); + } + + @Override + public CompleteMultipartUploadRequest readFrom( + Class type, Type genericType, + Annotation[] annotations, MediaType mediaType, + MultivaluedMap httpHeaders, InputStream entityStream) + throws IOException, WebApplicationException { + return new CompleteMultipartUploadRequest(); + } +}