HDDS-516. Implement CopyObject REST endpoint. Contributed by Bharat Viswanadham.
This commit is contained in:
Bharat Viswanadham
parent
c16c49b8c3
commit
021caaa55e
66
hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
vendored
Normal file
66
hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
vendored
Normal file
@ -0,0 +1,66 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
*** Settings ***
|
||||
Documentation S3 gateway test with aws cli
|
||||
Library OperatingSystem
|
||||
Library String
|
||||
Resource ../commonlib.robot
|
||||
Resource commonawslib.robot
|
||||
Test Setup Setup s3 tests
|
||||
|
||||
*** Variables ***
|
||||
${ENDPOINT_URL} http://s3g:9878
|
||||
${BUCKET} generated
|
||||
${DESTBUCKET} generated1
|
||||
|
||||
|
||||
*** Keywords ***
|
||||
Create Dest Bucket
|
||||
|
||||
${postfix} = Generate Random String 5 [NUMBERS]
|
||||
Set Suite Variable ${DESTBUCKET} destbucket-${postfix}
|
||||
Execute AWSS3APICli create-bucket --bucket ${DESTBUCKET}
|
||||
|
||||
|
||||
*** Test Cases ***
|
||||
Copy Object Happy Scenario
|
||||
Run Keyword if '${DESTBUCKET}' == 'generated1' Create Dest Bucket
|
||||
Execute date > /tmp/copyfile
|
||||
${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key copyobject/f1 --body /tmp/copyfile
|
||||
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix copyobject/
|
||||
Should contain ${result} f1
|
||||
|
||||
${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
|
||||
${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/
|
||||
Should contain ${result} f1
|
||||
#copying again will not throw error
|
||||
${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
|
||||
${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/
|
||||
Should contain ${result} f1
|
||||
|
||||
Copy Object Where Bucket is not available
|
||||
${result} = Execute AWSS3APICli and checkrc copy-object --bucket dfdfdfdfdfnonexistent --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 255
|
||||
Should contain ${result} NoSuchBucket
|
||||
${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source dfdfdfdfdfnonexistent/copyobject/f1 255
|
||||
Should contain ${result} NoSuchBucket
|
||||
|
||||
Copy Object Where both source and dest are same
|
||||
${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${DESTBUCKET}/copyobject/f1 255
|
||||
Should contain ${result} InvalidRequest
|
||||
|
||||
Copy Object Where Key not available
|
||||
${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/nonnonexistentkey 255
|
||||
Should contain ${result} NoSuchKey
|
||||
@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.ozone.s3.endpoint;
|
||||
|
||||
import org.apache.hadoop.ozone.s3.commontypes.IsoDateAdapter;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlElement;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* Copy object Response.
|
||||
*/
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
@XmlRootElement(name = "ListAllMyBucketsResult",
|
||||
namespace = "http://s3.amazonaws.com/doc/2006-03-01/")
|
||||
public class CopyObjectResponse {
|
||||
|
||||
@XmlJavaTypeAdapter(IsoDateAdapter.class)
|
||||
@XmlElement(name = "LastModified")
|
||||
private Instant lastModified;
|
||||
|
||||
@XmlElement(name = "ETag")
|
||||
private String eTag;
|
||||
|
||||
|
||||
public Instant getLastModified() {
|
||||
return lastModified;
|
||||
}
|
||||
|
||||
public void setLastModified(Instant lastModified) {
|
||||
this.lastModified = lastModified;
|
||||
}
|
||||
|
||||
public String getETag() {
|
||||
return eTag;
|
||||
}
|
||||
|
||||
public void setETag(String tag) {
|
||||
this.eTag = tag;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -41,11 +41,8 @@
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdds.client.ReplicationFactor;
|
||||
import org.apache.hadoop.hdds.client.ReplicationType;
|
||||
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
|
||||
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
|
||||
import org.apache.hadoop.ozone.client.OzoneBucket;
|
||||
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
|
||||
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
|
||||
@ -56,6 +53,7 @@
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@ -97,18 +95,27 @@ public Response put(
|
||||
ReplicationType replicationType,
|
||||
@DefaultValue("ONE") @QueryParam("replicationFactor")
|
||||
ReplicationFactor replicationFactor,
|
||||
@DefaultValue("32 * 1024 * 1024") @QueryParam("chunkSize")
|
||||
String chunkSize,
|
||||
@HeaderParam("Content-Length") long length,
|
||||
InputStream body) throws IOException, OS3Exception {
|
||||
|
||||
OzoneOutputStream output = null;
|
||||
try {
|
||||
Configuration config = new OzoneConfiguration();
|
||||
config.set(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
|
||||
String copyHeader = headers.getHeaderString("x-amz-copy-source");
|
||||
|
||||
if (copyHeader != null) {
|
||||
//Copy object, as copy source available.
|
||||
CopyObjectResponse copyObjectResponse = copyObject(
|
||||
copyHeader, bucketName, keyPath, replicationType,
|
||||
replicationFactor);
|
||||
return Response.status(Status.OK).entity(copyObjectResponse).header(
|
||||
"Connection", "close").build();
|
||||
}
|
||||
|
||||
// Normal put object
|
||||
OzoneBucket bucket = getBucket(bucketName);
|
||||
OzoneOutputStream output = bucket
|
||||
.createKey(keyPath, length, replicationType, replicationFactor);
|
||||
|
||||
output = bucket.createKey(keyPath, length, replicationType,
|
||||
replicationFactor);
|
||||
|
||||
if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
.equals(headers.getHeaderString("x-amz-content-sha256"))) {
|
||||
@ -116,13 +123,16 @@ public Response put(
|
||||
}
|
||||
|
||||
IOUtils.copy(body, output);
|
||||
output.close();
|
||||
|
||||
return Response.ok().status(HttpStatus.SC_OK)
|
||||
.build();
|
||||
} catch (IOException ex) {
|
||||
LOG.error("Exception occurred in PutObject", ex);
|
||||
throw ex;
|
||||
} finally {
|
||||
if (output != null) {
|
||||
output.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -239,4 +249,86 @@ public Response delete(
|
||||
public void setHeaders(HttpHeaders headers) {
|
||||
this.headers = headers;
|
||||
}
|
||||
|
||||
private CopyObjectResponse copyObject(String copyHeader,
|
||||
String destBucket,
|
||||
String destkey,
|
||||
ReplicationType replicationType,
|
||||
ReplicationFactor replicationFactor)
|
||||
throws OS3Exception, IOException {
|
||||
|
||||
if (copyHeader.startsWith("/")) {
|
||||
copyHeader = copyHeader.substring(1);
|
||||
}
|
||||
int pos = copyHeader.indexOf("/");
|
||||
if (pos == -1) {
|
||||
OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
|
||||
.INVALID_ARGUMENT, copyHeader);
|
||||
ex.setErrorMessage("Copy Source must mention the source bucket and " +
|
||||
"key: sourcebucket/sourcekey");
|
||||
throw ex;
|
||||
}
|
||||
String sourceBucket = copyHeader.substring(0, pos);
|
||||
String sourceKey = copyHeader.substring(pos + 1);
|
||||
|
||||
OzoneInputStream sourceInputStream = null;
|
||||
OzoneOutputStream destOutputStream = null;
|
||||
boolean closed = false;
|
||||
try {
|
||||
// Checking whether we trying to copying to it self.
|
||||
if (sourceBucket.equals(destBucket)) {
|
||||
if (sourceKey.equals(destkey)) {
|
||||
OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
|
||||
.INVALID_REQUEST, copyHeader);
|
||||
ex.setErrorMessage("This copy request is illegal because it is " +
|
||||
"trying to copy an object to it self itself without changing " +
|
||||
"the object's metadata, storage class, website redirect " +
|
||||
"location or encryption attributes.");
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
OzoneBucket sourceOzoneBucket = getBucket(sourceBucket);
|
||||
OzoneBucket destOzoneBucket = getBucket(destBucket);
|
||||
|
||||
OzoneKeyDetails sourceKeyDetails = sourceOzoneBucket.getKey(sourceKey);
|
||||
long sourceKeyLen = sourceKeyDetails.getDataSize();
|
||||
|
||||
sourceInputStream = sourceOzoneBucket.readKey(sourceKey);
|
||||
|
||||
destOutputStream = destOzoneBucket.createKey(destkey, sourceKeyLen,
|
||||
replicationType, replicationFactor);
|
||||
|
||||
IOUtils.copy(sourceInputStream, destOutputStream);
|
||||
|
||||
// Closing here, as if we don't call close this key will not commit in
|
||||
// OM, and getKey fails.
|
||||
sourceInputStream.close();
|
||||
destOutputStream.close();
|
||||
closed = true;
|
||||
|
||||
OzoneKeyDetails destKeyDetails = destOzoneBucket.getKey(destkey);
|
||||
|
||||
CopyObjectResponse copyObjectResponse = new CopyObjectResponse();
|
||||
copyObjectResponse.setETag(OzoneUtils.getRequestID());
|
||||
copyObjectResponse.setLastModified(Instant.ofEpochMilli(destKeyDetails
|
||||
.getModificationTime()));
|
||||
return copyObjectResponse;
|
||||
} catch (IOException ex) {
|
||||
if (ex.getMessage().contains("KEY_NOT_FOUND")) {
|
||||
throw S3ErrorTable.newError(S3ErrorTable.NO_SUCH_KEY, sourceKey);
|
||||
}
|
||||
LOG.error("Exception occurred in PutObject", ex);
|
||||
throw ex;
|
||||
} finally {
|
||||
if (!closed) {
|
||||
if (sourceInputStream != null) {
|
||||
sourceInputStream.close();
|
||||
}
|
||||
if (destOutputStream != null) {
|
||||
destOutputStream.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -52,6 +52,12 @@ private S3ErrorTable() {
|
||||
public static final OS3Exception NO_SUCH_KEY = new OS3Exception(
|
||||
"NoSuchKey", "The specified key does not exist", HTTP_NOT_FOUND);
|
||||
|
||||
public static final OS3Exception INVALID_ARGUMENT = new OS3Exception(
|
||||
"InvalidArgument", "Invalid Argument", HTTP_BAD_REQUEST);
|
||||
|
||||
public static final OS3Exception INVALID_REQUEST = new OS3Exception(
|
||||
"InvalidRequest", "Invalid Request", HTTP_BAD_REQUEST);
|
||||
|
||||
/**
|
||||
* Create a new instance of Error.
|
||||
* @param e Error Template
|
||||
|
||||
@ -38,6 +38,8 @@
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
@ -48,6 +50,9 @@ public class TestPutObject {
|
||||
private String userName = "ozone";
|
||||
private String bucketName = "b1";
|
||||
private String keyName = "key1";
|
||||
private String destBucket = "b2";
|
||||
private String destkey = "key2";
|
||||
private String nonexist = "nonexist";
|
||||
private OzoneClientStub clientStub;
|
||||
private ObjectStore objectStoreStub;
|
||||
private ObjectEndpoint objectEndpoint;
|
||||
@ -60,6 +65,7 @@ public void setup() throws IOException {
|
||||
|
||||
// Create bucket
|
||||
objectStoreStub.createS3Bucket(userName, bucketName);
|
||||
objectStoreStub.createS3Bucket("ozone1", destBucket);
|
||||
|
||||
// Create PutObject and setClient to OzoneClientStub
|
||||
objectEndpoint = new ObjectEndpoint();
|
||||
@ -75,8 +81,8 @@ public void testPutObject() throws IOException, OS3Exception {
|
||||
|
||||
//WHEN
|
||||
Response response = objectEndpoint.put(bucketName, keyName,
|
||||
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, "32 * 1024 * 1024",
|
||||
CONTENT.length(), body);
|
||||
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
|
||||
body);
|
||||
|
||||
//THEN
|
||||
String volumeName = clientStub.getObjectStore()
|
||||
@ -109,7 +115,6 @@ public void testPutObjectWithSignedChunks() throws IOException, OS3Exception {
|
||||
Response response = objectEndpoint.put(bucketName, keyName,
|
||||
ReplicationType.STAND_ALONE,
|
||||
ReplicationFactor.ONE,
|
||||
"32 * 1024 * 1024",
|
||||
chunkedContent.length(),
|
||||
new ByteArrayInputStream(chunkedContent.getBytes()));
|
||||
|
||||
@ -125,4 +130,107 @@ public void testPutObjectWithSignedChunks() throws IOException, OS3Exception {
|
||||
Assert.assertEquals(200, response.getStatus());
|
||||
Assert.assertEquals("1234567890abcde", keyContent);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCopyObject() throws IOException, OS3Exception {
|
||||
// Put object in to source bucket
|
||||
HttpHeaders headers = Mockito.mock(HttpHeaders.class);
|
||||
ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes());
|
||||
objectEndpoint.setHeaders(headers);
|
||||
keyName = "sourceKey";
|
||||
|
||||
Response response = objectEndpoint.put(bucketName, keyName,
|
||||
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
|
||||
body);
|
||||
|
||||
String volumeName = clientStub.getObjectStore().getOzoneVolumeName(
|
||||
bucketName);
|
||||
|
||||
OzoneInputStream ozoneInputStream = clientStub.getObjectStore().getVolume(
|
||||
volumeName).getBucket(bucketName).readKey(keyName);
|
||||
|
||||
String keyContent = IOUtils.toString(ozoneInputStream, Charset.forName(
|
||||
"UTF-8"));
|
||||
|
||||
Assert.assertEquals(200, response.getStatus());
|
||||
Assert.assertEquals(CONTENT, keyContent);
|
||||
|
||||
|
||||
// Add copy header, and then call put
|
||||
when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
|
||||
bucketName + "/" + keyName);
|
||||
|
||||
response = objectEndpoint.put(destBucket, destkey,
|
||||
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
|
||||
body);
|
||||
|
||||
// Check destination key and response
|
||||
volumeName = clientStub.getObjectStore().getOzoneVolumeName(destBucket);
|
||||
ozoneInputStream = clientStub.getObjectStore().getVolume(volumeName)
|
||||
.getBucket(destBucket).readKey(destkey);
|
||||
|
||||
keyContent = IOUtils.toString(ozoneInputStream, Charset.forName("UTF-8"));
|
||||
|
||||
Assert.assertEquals(200, response.getStatus());
|
||||
Assert.assertEquals(CONTENT, keyContent);
|
||||
|
||||
// source and dest same
|
||||
try {
|
||||
objectEndpoint.put(bucketName, keyName, ReplicationType.STAND_ALONE,
|
||||
ReplicationFactor.ONE, CONTENT.length(), body);
|
||||
fail("test copy object failed");
|
||||
} catch (OS3Exception ex) {
|
||||
Assert.assertTrue(ex.getErrorMessage().contains("This copy request is " +
|
||||
"illegal"));
|
||||
}
|
||||
|
||||
// source bucket not found
|
||||
try {
|
||||
when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
|
||||
nonexist + "/" + keyName);
|
||||
response = objectEndpoint.put(destBucket, destkey,
|
||||
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
|
||||
body);
|
||||
fail("test copy object failed");
|
||||
} catch (OS3Exception ex) {
|
||||
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
|
||||
}
|
||||
|
||||
// dest bucket not found
|
||||
try {
|
||||
when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
|
||||
bucketName + "/" + keyName);
|
||||
response = objectEndpoint.put(nonexist, destkey,
|
||||
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
|
||||
body);
|
||||
fail("test copy object failed");
|
||||
} catch (OS3Exception ex) {
|
||||
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
|
||||
}
|
||||
|
||||
//Both source and dest bucket not found
|
||||
try {
|
||||
when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
|
||||
nonexist + "/" + keyName);
|
||||
response = objectEndpoint.put(nonexist, destkey,
|
||||
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
|
||||
body);
|
||||
fail("test copy object failed");
|
||||
} catch (OS3Exception ex) {
|
||||
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
|
||||
}
|
||||
|
||||
// source key not found
|
||||
try {
|
||||
when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
|
||||
bucketName + "/" + nonexist);
|
||||
response = objectEndpoint.put("nonexistent", keyName,
|
||||
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
|
||||
body);
|
||||
fail("test copy object failed");
|
||||
} catch (OS3Exception ex) {
|
||||
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user