diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java index 241716fb8a..dd52a5721f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java @@ -403,7 +403,7 @@ public interface OzoneClient { * * @throws IOException */ - OzoneKey getkeyDetails(String volumeName, String bucketName, + OzoneKey getKeyDetails(String volumeName, String bucketName, String keyName) throws IOException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java index 01807b0896..3c79778c3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java @@ -20,9 +20,16 @@ package org.apache.hadoop.ozone; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.ozone.protocol.proto + .ContainerProtos.ChunkInfo; +import org.apache.hadoop.hdfs.ozone.protocol.proto + .ContainerProtos.GetKeyResponseProto; +import org.apache.hadoop.hdfs.ozone.protocol.proto + .ContainerProtos.KeyData; import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ksm.helpers.KsmBucketArgs; import org.apache.hadoop.ksm.helpers.KsmBucketInfo; import org.apache.hadoop.ksm.helpers.KsmKeyArgs; import org.apache.hadoop.ksm.helpers.KsmKeyInfo; @@ -45,6 +52,7 @@ import org.apache.hadoop.scm.protocolPB .StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.scm.protocolPB .StorageContainerLocationProtocolPB; +import org.apache.hadoop.scm.storage.ChunkInputStream; import org.apache.hadoop.scm.storage.ChunkOutputStream; import org.apache.hadoop.scm.storage.ContainerProtocolCalls; import org.apache.hadoop.security.UserGroupInformation; @@ -375,34 +383,65 @@ public class OzoneClientImpl implements OzoneClient, Closeable { public void addBucketAcls(String volumeName, String bucketName, List addAcls) throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + Preconditions.checkNotNull(addAcls); + KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder(); + builder.setVolumeName(volumeName) + .setBucketName(bucketName) + .setAddAcls(addAcls); + keySpaceManagerClient.setBucketProperty(builder.build()); } @Override public void removeBucketAcls(String volumeName, String bucketName, List removeAcls) throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + Preconditions.checkNotNull(removeAcls); + KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder(); + builder.setVolumeName(volumeName) + .setBucketName(bucketName) + .setRemoveAcls(removeAcls); + keySpaceManagerClient.setBucketProperty(builder.build()); } @Override public void setBucketVersioning(String volumeName, String bucketName, Versioning versioning) throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + Preconditions.checkNotNull(versioning); + KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder(); + builder.setVolumeName(volumeName) + .setBucketName(bucketName) + .setIsVersionEnabled(getBucketVersioningFlag( + versioning)); + keySpaceManagerClient.setBucketProperty(builder.build()); } @Override public void setBucketStorageType(String volumeName, String bucketName, StorageType storageType) throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + Preconditions.checkNotNull(storageType); + KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder(); + builder.setVolumeName(volumeName) + .setBucketName(bucketName) + .setStorageType(storageType); + keySpaceManagerClient.setBucketProperty(builder.build()); } @Override public void deleteBucket(String volumeName, String bucketName) throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + keySpaceManagerClient.deleteBucket(volumeName, bucketName); } @Override @@ -480,14 +519,54 @@ public class OzoneClientImpl implements OzoneClient, Closeable { public OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + Preconditions.checkNotNull(keyName); + String requestId = UUID.randomUUID().toString(); + KsmKeyArgs keyArgs = new KsmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .build(); + KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs); + String containerKey = buildContainerKey(volumeName, + bucketName, keyName); + String containerName = keyInfo.getContainerName(); + XceiverClientSpi xceiverClient = getContainer(containerName); + boolean success = false; + try { + LOG.debug("get key accessing {} {}", + xceiverClient.getPipeline().getContainerName(), containerKey); + KeyData containerKeyData = KeyData.newBuilder().setContainerName( + xceiverClient.getPipeline().getContainerName()) + .setName(containerKey).build(); + GetKeyResponseProto response = ContainerProtocolCalls + .getKey(xceiverClient, containerKeyData, requestId); + List chunks = response.getKeyData().getChunksList(); + success = true; + return new OzoneInputStream(new ChunkInputStream( + containerKey, xceiverClientManager, xceiverClient, + chunks, requestId)); + } finally { + if (!success) { + xceiverClientManager.releaseClient(xceiverClient); + } + } } @Override public void deleteKey(String volumeName, String bucketName, String keyName) throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + Preconditions.checkNotNull(keyName); + KsmKeyArgs keyArgs = new KsmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .build(); + keySpaceManagerClient.deleteKey(keyArgs); } @Override @@ -498,7 +577,7 @@ public class OzoneClientImpl implements OzoneClient, Closeable { } @Override - public OzoneKey getkeyDetails(String volumeName, String bucketName, + public OzoneKey getKeyDetails(String volumeName, String bucketName, String keyName) throws IOException { Preconditions.checkNotNull(volumeName); @@ -514,6 +593,27 @@ public class OzoneClientImpl implements OzoneClient, Closeable { return new OzoneKey(keyInfo); } + /** + * Converts Versioning to boolean. + * + * @param version + * @return corresponding boolean value + */ + private boolean getBucketVersioningFlag( + Versioning version) { + if(version != null) { + switch(version) { + case ENABLED: + return true; + case DISABLED: + case NOT_DEFINED: + default: + return false; + } + } + return false; + } + @Override public void close() throws IOException { if(xceiverClientManager != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java index 30ef1449d2..b861f7dfe0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java @@ -19,14 +19,19 @@ package org.apache.hadoop.ozone; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.ozone.io.OzoneInputStream; import org.apache.hadoop.ozone.io.OzoneOutputStream; import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; /** @@ -34,6 +39,9 @@ import java.util.UUID; */ public class TestOzoneClientImpl { + @Rule + public ExpectedException thrown = ExpectedException.none(); + private static MiniOzoneCluster cluster = null; private static OzoneClientImpl ozClient = null; @@ -100,6 +108,38 @@ public class TestOzoneClientImpl { } } + @Test + public void testSetVolumeOwner() + throws IOException, OzoneException { + String volumeName = UUID.randomUUID().toString(); + ozClient.createVolume(volumeName); + ozClient.setVolumeOwner(volumeName, "test"); + OzoneVolume volume = ozClient.getVolumeDetails(volumeName); + Assert.assertEquals("test", volume.getOwnerName()); + } + + @Test + public void testSetVolumeQuota() + throws IOException, OzoneException { + String volumeName = UUID.randomUUID().toString(); + ozClient.createVolume(volumeName); + ozClient.setVolumeQuota(volumeName, 10000000000L); + OzoneVolume volume = ozClient.getVolumeDetails(volumeName); + Assert.assertEquals(10000000000L, volume.getQuota()); + } + + @Test + public void testDeleteVolume() + throws IOException, OzoneException { + thrown.expectMessage("Info Volume failed, error"); + String volumeName = UUID.randomUUID().toString(); + ozClient.createVolume(volumeName); + OzoneVolume volume = ozClient.getVolumeDetails(volumeName); + Assert.assertNotNull(volume); + ozClient.deleteVolume(volumeName); + ozClient.getVolumeDetails(volumeName); + } + @Test public void testCreateBucket() throws IOException, OzoneException { @@ -184,6 +224,85 @@ public class TestOzoneClientImpl { } } + @Test + public void testAddBucketAcl() + throws IOException, OzoneException { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + ozClient.createVolume(volumeName); + ozClient.createBucket(volumeName, bucketName); + List acls = new ArrayList<>(); + acls.add(new OzoneAcl( + OzoneAcl.OzoneACLType.USER, "test", + OzoneAcl.OzoneACLRights.READ_WRITE)); + ozClient.addBucketAcls(volumeName, bucketName, acls); + OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); + Assert.assertEquals(bucketName, bucket.getBucketName()); + Assert.assertTrue(bucket.getAcls().contains(acls.get(0))); + } + + @Test + public void testRemoveBucketAcl() + throws IOException, OzoneException { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + OzoneAcl userAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, "test", + OzoneAcl.OzoneACLRights.READ_WRITE); + ozClient.createVolume(volumeName); + ozClient.createBucket(volumeName, bucketName, userAcl); + List acls = new ArrayList<>(); + acls.add(userAcl); + ozClient.removeBucketAcls(volumeName, bucketName, acls); + OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); + Assert.assertEquals(bucketName, bucket.getBucketName()); + Assert.assertTrue(!bucket.getAcls().contains(acls.get(0))); + } + + @Test + public void testSetBucketVersioning() + throws IOException, OzoneException { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + ozClient.createVolume(volumeName); + ozClient.createBucket(volumeName, bucketName); + ozClient.setBucketVersioning(volumeName, bucketName, + OzoneConsts.Versioning.ENABLED); + OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); + Assert.assertEquals(bucketName, bucket.getBucketName()); + Assert.assertEquals(OzoneConsts.Versioning.ENABLED, + bucket.getVersioning()); + } + + @Test + public void testSetBucketStorageType() + throws IOException, OzoneException { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + ozClient.createVolume(volumeName); + ozClient.createBucket(volumeName, bucketName); + ozClient.setBucketStorageType(volumeName, bucketName, + StorageType.SSD); + OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); + Assert.assertEquals(bucketName, bucket.getBucketName()); + Assert.assertEquals(StorageType.SSD, bucket.getStorageType()); + } + + + @Test + public void testDeleteBucket() + throws IOException, OzoneException { + thrown.expectMessage("Info Bucket failed, error"); + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + ozClient.createVolume(volumeName); + ozClient.createBucket(volumeName, bucketName); + OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); + Assert.assertNotNull(bucket); + ozClient.deleteBucket(volumeName, bucketName); + ozClient.getBucketDetails(volumeName, bucketName); + } + + @Test public void testPutKey() throws IOException, OzoneException { @@ -196,9 +315,33 @@ public class TestOzoneClientImpl { OzoneOutputStream out = ozClient.createKey(volumeName, bucketName, keyName, value.getBytes().length); out.write(value.getBytes()); - OzoneKey key = ozClient.getkeyDetails(volumeName, bucketName, keyName); + out.close(); + OzoneKey key = ozClient.getKeyDetails(volumeName, bucketName, keyName); Assert.assertEquals(keyName, key.getKeyName()); - //Content validation has to be done after getKey implementation. + OzoneInputStream is = ozClient.getKey(volumeName, bucketName, keyName); + byte[] fileContent = new byte[value.getBytes().length]; + is.read(fileContent); + Assert.assertEquals(value, new String(fileContent)); + } + + @Test + public void testDeleteKey() + throws IOException, OzoneException { + thrown.expectMessage("Lookup key failed, error"); + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + String value = "sample value"; + ozClient.createVolume(volumeName); + ozClient.createBucket(volumeName, bucketName); + OzoneOutputStream out = ozClient.createKey(volumeName, bucketName, + keyName, value.getBytes().length); + out.write(value.getBytes()); + out.close(); + OzoneKey key = ozClient.getKeyDetails(volumeName, bucketName, keyName); + Assert.assertEquals(keyName, key.getKeyName()); + ozClient.deleteKey(volumeName, bucketName, keyName); + ozClient.getKeyDetails(volumeName, bucketName, keyName); } /**