diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index 419655602d..1ea5c03623 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -94,6 +94,15 @@ public void createS3Bucket(String userName, String s3BucketName) throws proxy.createS3Bucket(userName, s3BucketName); } + /** + * Deletes an s3 bucket and removes mapping of Ozone volume/bucket. + * @param bucketName - S3 Bucket Name. + * @throws IOException in case the bucket cannot be deleted. + */ + public void deleteS3Bucket(String bucketName) throws IOException { + proxy.deleteS3Bucket(bucketName); + } + /** * Returns the Ozone Namespace for the S3Bucket. It will return the * OzoneVolume/OzoneBucketName. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index b750a5a624..2737a9c923 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -330,6 +330,14 @@ OzoneKeyDetails getKeyDetails(String volumeName, String bucketName, */ void createS3Bucket(String userName, String s3BucketName) throws IOException; + /** + * Deletes an s3 bucket and removes mapping of Ozone volume/bucket. + * @param bucketName - S3 Bucket Name. + * @throws IOException in case the bucket cannot be deleted. + */ + void deleteS3Bucket(String bucketName) throws IOException; + + /** * Returns the Ozone Namespace for the S3Bucket. It will return the * OzoneVolume/OzoneBucketName. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java index 8e7590373c..b6e2f09fbd 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java @@ -827,6 +827,13 @@ public void createS3Bucket(String userName, String s3BucketName) "support this operation."); } + @Override + public void deleteS3Bucket(String s3BucketName) + throws IOException { + throw new UnsupportedOperationException("Ozone REST protocol does not " + + "support this operation."); + } + @Override public String getOzoneBucketMapping(String s3BucketName) throws IOException { throw new UnsupportedOperationException("Ozone REST protocol does not " + diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 11b5474f71..734d2a6ca4 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -579,6 +579,14 @@ public void createS3Bucket(String userName, String s3BucketName) ozoneManagerClient.createS3Bucket(userName, s3BucketName); } + @Override + public void deleteS3Bucket(String s3BucketName) + throws IOException { + Preconditions.checkArgument(Strings.isNotBlank(s3BucketName), "bucket " + + "name cannot be null or empty."); + ozoneManagerClient.deleteS3Bucket(s3BucketName); + } + @Override public String getOzoneBucketMapping(String s3BucketName) throws IOException { Preconditions.checkArgument(Strings.isNotBlank(s3BucketName), "bucket " + diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index d31f70b7b6..c021e64c6b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -264,6 +264,13 @@ List listKeys(String volumeName, */ void createS3Bucket(String userName, String s3BucketName) throws IOException; + /** + * Delets an S3 bucket inside Ozone manager and deletes the mapping. + * @param s3BucketName - S3 bucket Name. + * @throws IOException in case the bucket cannot be deleted. + */ + void deleteS3Bucket(String s3BucketName) throws IOException; + /** * Returns the Ozone Namespace for the S3Bucket. It will return the * OzoneVolume/OzoneBucketName. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index dfe3c74688..94c57e5d39 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -118,6 +118,10 @@ .OzoneManagerProtocolProtos.S3BucketRequest; import org.apache.hadoop.ozone.protocol.proto .OzoneManagerProtocolProtos.S3BucketResponse; +import org.apache.hadoop.ozone.protocol.proto + .OzoneManagerProtocolProtos.S3DeleteBucketRequest; +import org.apache.hadoop.ozone.protocol.proto + .OzoneManagerProtocolProtos.S3DeleteBucketResponse; import org.apache.hadoop.ozone.protocol.proto .OzoneManagerProtocolProtos.S3BucketInfoRequest; import org.apache.hadoop.ozone.protocol.proto @@ -795,6 +799,25 @@ public void createS3Bucket(String userName, String s3BucketName) } + @Override + public void deleteS3Bucket(String s3BucketName) throws IOException { + S3DeleteBucketRequest request = S3DeleteBucketRequest.newBuilder() + .setS3BucketName(s3BucketName) + .build(); + final S3DeleteBucketResponse resp; + try { + resp = rpcProxy.deleteS3Bucket(NULL_RPC_CONTROLLER, request); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + + if(resp.getStatus() != Status.OK) { + throw new IOException("Creating S3 bucket failed, error: " + + resp.getStatus()); + } + + } + @Override public String getOzoneBucketMapping(String s3BucketName) throws IOException { diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto index eef19a5bff..19fb204fd2 100644 --- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto @@ -381,6 +381,14 @@ message S3BucketInfoResponse { optional string ozoneMapping = 2; } +message S3DeleteBucketRequest { + required string s3bucketName = 1; +} + +message S3DeleteBucketResponse { + required Status status = 1; +} + /** The OM service that takes care of Ozone namespace. @@ -506,6 +514,9 @@ service OzoneManagerService { rpc createS3Bucket(S3BucketRequest) returns(S3BucketResponse); + rpc deleteS3Bucket(S3DeleteBucketRequest) + returns(S3DeleteBucketResponse); + /** Gets the Ozone Mapping information for the S3Bucket. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java index cbc29f5b11..ee8cc08579 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.protocolPB. StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.junit.AfterClass; import org.junit.Assert; @@ -215,6 +216,33 @@ public void testCreateS3Bucket() Assert.assertTrue(volume.getCreationTime() >= currentTime); } + @Test + public void testDeleteS3Bucket() + throws IOException, OzoneException { + long currentTime = Time.now(); + String userName = "ozone1"; + String bucketName = UUID.randomUUID().toString(); + store.createS3Bucket(userName, bucketName); + String volumeName = store.getOzoneVolumeName(bucketName); + OzoneVolume volume = store.getVolume(volumeName); + OzoneBucket bucket = volume.getBucket(bucketName); + Assert.assertEquals(bucketName, bucket.getName()); + Assert.assertTrue(bucket.getCreationTime() >= currentTime); + Assert.assertTrue(volume.getCreationTime() >= currentTime); + store.deleteS3Bucket(bucketName); + thrown.expect(IOException.class); + store.getOzoneVolumeName(bucketName); + } + + @Test + public void testDeleteS3NonExistingBucket() { + try { + store.deleteS3Bucket(UUID.randomUUID().toString()); + } catch (IOException ex) { + GenericTestUtils.assertExceptionContains("NOT_FOUND", ex); + } + } + @Test public void testCreateS3BucketMapping() throws IOException, OzoneException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index dbcd401d17..0d04f59a66 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -1140,6 +1140,15 @@ public void createS3Bucket(String userName, String s3BucketName) s3BucketManager.createS3Bucket(userName, s3BucketName); } + @Override + /** + * {@inheritDoc} + */ + public void deleteS3Bucket(String s3BucketName) + throws IOException { + s3BucketManager.deleteS3Bucket(s3BucketName); + } + @Override /** * {@inheritDoc} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java index 5fb4c60c93..4144662de8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java @@ -33,6 +33,13 @@ public interface S3BucketManager { */ void createS3Bucket(String userName, String bucketName) throws IOException; + /** + * Deletes an s3 bucket and removes mapping of Ozone volume/bucket. + * @param bucketName - S3 Bucket Name. + * @throws IOException in case the bucket cannot be deleted. + */ + void deleteS3Bucket(String bucketName) throws IOException; + /** * Returns the Ozone volume/bucket where the S3 Bucket points to. * @param s3BucketName - S3 Bucket Name diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java index 89c529a0d7..e64995f4cd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java @@ -127,6 +127,30 @@ public void createS3Bucket(String userName, String bucketName) } } + @Override + public void deleteS3Bucket(String bucketName) throws IOException { + Preconditions.checkArgument( + Strings.isNotBlank(bucketName), "Bucket name cannot be null or empty"); + + omMetadataManager.getLock().acquireS3Lock(bucketName); + try { + byte[] bucket = bucketName.getBytes(StandardCharsets.UTF_8); + byte[] map = omMetadataManager.getS3Table().get(bucket); + + if (map == null) { + throw new OMException("No such S3 bucket. " + bucketName, + OMException.ResultCodes.S3_BUCKET_NOT_FOUND); + } + bucketManager.deleteBucket(getOzoneVolumeName(bucketName), bucketName); + omMetadataManager.getS3Table().delete(bucket); + } catch(IOException ex) { + throw ex; + } finally { + omMetadataManager.getLock().releaseS3Lock(bucketName); + } + + } + private String formatOzoneVolumeName(String userName) { return String.format("s3%s", userName); } @@ -202,4 +226,5 @@ public String getOzoneBucketName(String s3BucketName) throws IOException { String mapping = getOzoneBucketMapping(s3BucketName); return mapping.split("/")[1]; } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java index 72ad9cb404..9416056b97 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java @@ -97,6 +97,10 @@ .S3BucketRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .S3BucketResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .S3DeleteBucketRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .S3DeleteBucketResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .ServiceListRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos @@ -596,6 +600,20 @@ public S3BucketResponse createS3Bucket(RpcController controller, return resp.build(); } + @Override + public S3DeleteBucketResponse deleteS3Bucket(RpcController controller, + S3DeleteBucketRequest request) throws + ServiceException { + S3DeleteBucketResponse.Builder resp = S3DeleteBucketResponse.newBuilder(); + try { + impl.deleteS3Bucket(request.getS3BucketName()); + resp.setStatus(Status.OK); + } catch (IOException e) { + resp.setStatus(exceptionToResponseStatus(e)); + } + return resp.build(); + } + @Override public S3BucketInfoResponse getS3Bucketinfo(RpcController controller, S3BucketInfoRequest request) throws ServiceException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java index 4837f9a02e..75349fb080 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java @@ -75,6 +75,24 @@ public void testCreateS3Bucket() throws IOException { } + @Test + public void testDeleteS3Bucket() throws IOException { + S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr, + volumeManager, bucketManager); + s3BucketManager.createS3Bucket("ozone", "s3bucket"); + + // This call should have created a ozone volume called s3ozone and bucket + // called s3ozone/s3bucket. + Assert.assertNotNull(volumeManager.getVolumeInfo("s3ozone")); + Assert.assertNotNull(bucketManager.getBucketInfo("s3ozone", "s3bucket")); + + s3BucketManager.deleteS3Bucket("s3bucket"); + + //Deleting non existing bucket should throw. + thrown.expect(IOException.class); + s3BucketManager.deleteS3Bucket("s3bucket"); + } + @Test public void testGetS3BucketMapping() throws IOException { S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr, diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java index fb523c7dd5..daa75a9c1a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java @@ -107,6 +107,49 @@ protected String createS3Bucket(String userName, String bucketName) throws return "/"+location; } + /** + * Deletes an s3 bucket and removes mapping of Ozone volume/bucket. + * @param s3BucketName - S3 Bucket Name. + * @throws IOException in case the bucket cannot be deleted. + */ + public void deleteS3Bucket(String s3BucketName) + throws IOException { + client.getObjectStore().deleteS3Bucket(s3BucketName); + } + + /** + * Returns the Ozone Namespace for the S3Bucket. It will return the + * OzoneVolume/OzoneBucketName. + * @param s3BucketName - S3 Bucket Name. + * @return String - The Ozone canonical name for this s3 bucket. This + * string is useful for mounting an OzoneFS. + * @throws IOException - Error is throw if the s3bucket does not exist. + */ + public String getOzoneBucketMapping(String s3BucketName) throws IOException { + return client.getObjectStore().getOzoneBucketMapping(s3BucketName); + } + + /** + * Returns the corresponding Ozone volume given an S3 Bucket. + * @param s3BucketName - S3Bucket Name. + * @return String - Ozone Volume name. + * @throws IOException - Throws if the s3Bucket does not exist. + */ + public String getOzoneVolumeName(String s3BucketName) throws IOException { + return client.getObjectStore().getOzoneVolumeName(s3BucketName); + } + + /** + * Returns the corresponding Ozone bucket name for the given S3 bucket. + * @param s3BucketName - S3Bucket Name. + * @return String - Ozone bucket Name. + * @throws IOException - Throws if the s3bucket does not exist. + */ + public String getOzoneBucketName(String s3BucketName) throws IOException { + return client.getObjectStore().getOzoneBucketName(s3BucketName); + } + + @VisibleForTesting public void setClient(OzoneClient ozoneClient) { this.client = ozoneClient; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java index 0777856c85..f0c52912e5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java @@ -36,6 +36,7 @@ public ObjectStoreStub() { } private Map volumes = new HashMap<>(); + private Map bucketVolumeMap = new HashMap<>(); @Override public void createVolume(String volumeName) throws IOException { @@ -107,4 +108,29 @@ public Iterator listVolumesByUser(String user, public void deleteVolume(String volumeName) throws IOException { volumes.remove(volumeName); } + + @Override + public void createS3Bucket(String userName, String s3BucketName) throws + IOException { + bucketVolumeMap.put(s3BucketName, "s3"+userName+"/"+s3BucketName); + } + + @Override + public void deleteS3Bucket(String s3BucketName) throws + IOException { + bucketVolumeMap.remove(s3BucketName); + } + + @Override + public String getOzoneBucketMapping(String s3BucketName) throws IOException { + return bucketVolumeMap.get(s3BucketName); + } + @Override + public String getOzoneVolumeName(String s3BucketName) throws IOException { + return bucketVolumeMap.get(s3BucketName).split("/")[0]; + } + @Override + public String getOzoneBucketName(String s3BucketName) throws IOException { + return bucketVolumeMap.get(s3BucketName).split("/")[1]; + } }