diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java index 4de07aa814..a49137a7cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java @@ -54,6 +54,10 @@ public final class KsmBucketInfo { * [RAM_DISK, SSD, DISK, ARCHIVE] */ private StorageType storageType; + /** + * Creation time of bucket. + */ + private final long creationTime; /** * Private constructor, constructed via builder. @@ -62,15 +66,17 @@ public final class KsmBucketInfo { * @param acls - list of ACLs. * @param isVersionEnabled - Bucket version flag. * @param storageType - Storage type to be used. + * @param creationTime - Bucket creation time. */ private KsmBucketInfo(String volumeName, String bucketName, List acls, boolean isVersionEnabled, - StorageType storageType) { + StorageType storageType, long creationTime) { this.volumeName = volumeName; this.bucketName = bucketName; this.acls = acls; this.isVersionEnabled = isVersionEnabled; this.storageType = storageType; + this.creationTime = creationTime; } /** @@ -113,6 +119,15 @@ public StorageType getStorageType() { return storageType; } + /** + * Returns creation time. + * + * @return long + */ + public long getCreationTime() { + return creationTime; + } + /** * Returns new builder class that builds a KsmBucketInfo. * @@ -131,6 +146,7 @@ public static class Builder { private List acls; private Boolean isVersionEnabled; private StorageType storageType; + private long creationTime; Builder() { //Default values @@ -164,6 +180,11 @@ public Builder setStorageType(StorageType storage) { return this; } + public Builder setCreationTime(long createdOn) { + this.creationTime = createdOn; + return this; + } + /** * Constructs the KsmBucketInfo. * @return instance of KsmBucketInfo. @@ -174,8 +195,9 @@ public KsmBucketInfo build() { Preconditions.checkNotNull(acls); Preconditions.checkNotNull(isVersionEnabled); Preconditions.checkNotNull(storageType); + return new KsmBucketInfo(volumeName, bucketName, acls, - isVersionEnabled, storageType); + isVersionEnabled, storageType, creationTime); } } @@ -191,6 +213,7 @@ public BucketInfo getProtobuf() { .setIsVersionEnabled(isVersionEnabled) .setStorageType(PBHelperClient.convertStorageType( storageType)) + .setCreationTime(creationTime) .build(); } @@ -207,6 +230,6 @@ public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) { KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), bucketInfo.getIsVersionEnabled(), PBHelperClient.convertStorageType( - bucketInfo.getStorageType())); + bucketInfo.getStorageType()), bucketInfo.getCreationTime()); } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto index db364beaa8..5f28e585c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto @@ -162,6 +162,7 @@ message BucketInfo { repeated OzoneAclInfo acls = 3; required bool isVersionEnabled = 4 [default = false]; required StorageTypeProto storageType = 5 [default = DISK]; + required uint64 creationTime = 6; } message BucketArgs { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java index 585dcff209..95f76dd287 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java @@ -202,6 +202,7 @@ public void setBucketProperty(KsmBucketArgs args) throws IOException { bucketInfoBuilder.setIsVersionEnabled( oldBucketInfo.getIsVersionEnabled()); } + bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime()); metadataManager.put(bucketKey, bucketInfoBuilder.build() .getProtobuf().toByteArray()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java index 1ebf509963..1dab4aa75f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java @@ -180,6 +180,7 @@ public void setOwner(String volume, String owner) throws IOException { .setAdminName(volumeArgs.getAdminName()) .setOwnerName(owner) .setQuotaInBytes(volumeArgs.getQuotaInBytes()) + .setCreationTime(volumeArgs.getCreationTime()) .build(); VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf(); @@ -221,6 +222,7 @@ public void setQuota(String volume, long quota) throws IOException { .setAdminName(volumeArgs.getAdminName()) .setOwnerName(volumeArgs.getOwnerName()) .setQuotaInBytes(quota) + .setCreationTime(volumeArgs.getCreationTime()) .build(); VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java index e92a0d2555..a83a720510 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java @@ -146,6 +146,15 @@ public StorageType getStorageType() { return bucketInfo.getStorageType(); } + /** + * Gets the creation time of the bucket. + * + * @return String + */ + public String getCreatedOn() { + return bucketInfo.getCreatedOn(); + } + /** * Puts an Object in Ozone bucket. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java index 53c71199df..e66cd204df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java @@ -62,6 +62,7 @@ public class BucketInfo implements Comparable { private String volumeName; private String bucketName; + private String createdOn; private List acls; private OzoneConsts.Versioning versioning; private StorageType storageType; @@ -173,6 +174,24 @@ public void setBucketName(String bucketName) { this.bucketName = bucketName; } + /** + * Sets creation time of the bucket. + * + * @param creationTime - Date String + */ + public void setCreatedOn(String creationTime) { + this.createdOn = creationTime; + } + + /** + * Returns creation time. + * + * @return creation time of bucket. + */ + public String getCreatedOn() { + return createdOn; + } + /** * Returns a JSON string of this object. * After stripping out bytesUsed and keyCount diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java index 76c9be0d30..f2e21806fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java @@ -239,6 +239,7 @@ public void createBucket(final BucketArgs args) builder.setIsVersionEnabled(getBucketVersioningProtobuf( args.getVersioning())); } + builder.setCreationTime(Time.now()); keySpaceManagerClient.createBucket(builder.build()); } @@ -349,6 +350,7 @@ public ListBuckets listBuckets(ListArgs args) bk.setBucketName(bucketInfo.getBucketName()); bk.setStorageType(bucketInfo.getStorageType()); bk.setAcls(bucketInfo.getAcls()); + bk.setCreatedOn(OzoneUtils.formatTime(bucketInfo.getCreationTime())); result.addBucket(bk); } return result; @@ -375,6 +377,8 @@ public BucketInfo getBucketInfo(BucketArgs args) } bucketInfo.setStorageType(ksmBucketInfo.getStorageType()); bucketInfo.setAcls(ksmBucketInfo.getAcls()); + bucketInfo.setCreatedOn( + OzoneUtils.formatTime(ksmBucketInfo.getCreationTime())); return bucketInfo; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java index a41fc00ac9..3470c8ac7e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java @@ -408,7 +408,11 @@ public void testInfoBucket() throws Exception { String[] args = new String[] {"-infoBucket", url + "/" + vol.getVolumeName() + "/" + bucketName}; assertEquals(0, ToolRunner.run(shell, args)); - assertTrue(out.toString().contains(bucketName)); + + String output = out.toString(); + assertTrue(output.contains(bucketName)); + assertTrue(output.contains("createdOn") + && output.contains(OzoneConsts.OZONE_TIME_ZONE)); // test get info from a non-exist bucket args = new String[] {"-infoBucket", @@ -429,6 +433,9 @@ public void testUpdateBucket() throws Exception { url + "/" + vol.getVolumeName() + "/" + bucketName, "-addAcl", "user:frodo:rw,group:samwise:r"}; assertEquals(0, ToolRunner.run(shell, args)); + String output = out.toString(); + assertTrue(output.contains("createdOn") + && output.contains(OzoneConsts.OZONE_TIME_ZONE)); bucket = vol.getBucket(bucketName); assertEquals(2, bucket.getAcls().size()); @@ -480,8 +487,11 @@ public void testListBucket() throws Exception { List volumes = getValueLines("volumeName", out.toString()); List buckets = getValueLines("bucketName", out.toString()); + List creationTimes = getValueLines("createdOn", out.toString()); assertEquals(11, volumes.size()); assertEquals(11, buckets.size()); + assertEquals(11, creationTimes.size()); + // sort bucket names since the return buckets isn't in created order Collections.sort(bucketNames); // return bucket names should be [test-bucket0, test-bucket1, @@ -489,6 +499,7 @@ public void testListBucket() throws Exception { for (int i = 0; i < buckets.size(); i++) { assertTrue(buckets.get(i).contains(bucketNames.get(i))); assertTrue(volumes.get(i).contains(vol.getVolumeName())); + assertTrue(creationTimes.get(i).contains(OzoneConsts.OZONE_TIME_ZONE)); } out.reset(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java index 0be5df6b13..a288bdabab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java @@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.web.request.OzoneQuota; import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; @@ -35,10 +36,12 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.text.ParseException; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class TestBuckets { @@ -90,12 +93,12 @@ public static void shutdown() { } @Test - public void testCreateBucket() throws OzoneException, IOException { + public void testCreateBucket() throws Exception { runTestCreateBucket(ozoneRestClient); } static void runTestCreateBucket(OzoneRestClient client) - throws OzoneException, IOException { + throws OzoneException, IOException, ParseException { String volumeName = OzoneUtils.getRequestID().toLowerCase(); client.setUserAuth("hdfs"); OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB"); @@ -103,10 +106,15 @@ static void runTestCreateBucket(OzoneRestClient client) // create 10 buckets under same volume for (int x = 0; x < 10; x++) { + long currentTime = Time.now(); String bucketName = OzoneUtils.getRequestID().toLowerCase(); OzoneBucket bucket = vol.createBucket(bucketName, acls, StorageType.DEFAULT); assertEquals(bucket.getBucketName(), bucketName); + + // verify the bucket creation time + assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn()) + / 1000) >= (currentTime / 1000)); } client.close(); @@ -118,12 +126,12 @@ static void runTestCreateBucket(OzoneRestClient client) } @Test - public void testAddBucketAcls() throws OzoneException, IOException { + public void testAddBucketAcls() throws Exception { runTestAddBucketAcls(ozoneRestClient); } static void runTestAddBucketAcls(OzoneRestClient client) - throws OzoneException, IOException { + throws OzoneException, IOException, ParseException { String volumeName = OzoneUtils.getRequestID().toLowerCase(); client.setUserAuth("hdfs"); OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB"); @@ -133,16 +141,19 @@ static void runTestAddBucketAcls(OzoneRestClient client) vol.addAcls(bucketName, acls); OzoneBucket updatedBucket = vol.getBucket(bucketName); assertEquals(updatedBucket.getAcls().size(), 2); + // verify if the creation time is missing after update operation + assertTrue( + (OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0); client.close(); } @Test - public void testRemoveBucketAcls() throws OzoneException, IOException { + public void testRemoveBucketAcls() throws Exception { runTestRemoveBucketAcls(ozoneRestClient); } static void runTestRemoveBucketAcls(OzoneRestClient client) - throws OzoneException, IOException { + throws OzoneException, IOException, ParseException { String volumeName = OzoneUtils.getRequestID().toLowerCase(); client.setUserAuth("hdfs"); OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB"); @@ -155,6 +166,9 @@ static void runTestRemoveBucketAcls(OzoneRestClient client) // We removed all acls assertEquals(updatedBucket.getAcls().size(), 0); + // verify if the creation time is missing after update operation + assertTrue( + (OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0); client.close(); } @@ -183,16 +197,18 @@ static void runTestDeleteBucket(OzoneRestClient client) } @Test - public void testListBucket() throws OzoneException, IOException { + public void testListBucket() throws Exception { runTestListBucket(ozoneRestClient); } static void runTestListBucket(OzoneRestClient client) - throws OzoneException, IOException { + throws OzoneException, IOException, ParseException { String volumeName = OzoneUtils.getRequestID().toLowerCase(); client.setUserAuth("hdfs"); OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB"); String[] acls = {"user:frodo:rw", "user:samwise:rw"}; + + long currentTime = Time.now(); for (int x = 0; x < 10; x++) { String bucketName = "listbucket-test-" + x; vol.createBucket(bucketName, acls); @@ -200,6 +216,11 @@ static void runTestListBucket(OzoneRestClient client) List bucketList = vol.listBuckets("100", null, null); assertEquals(bucketList.size(), 10); + for (OzoneBucket bucket : bucketList) { + assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn()) + / 1000) >= (currentTime / 1000)); + } + bucketList = vol.listBuckets("3", null, null); assertEquals(bucketList.size(), 3); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBucketsRatis.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBucketsRatis.java index 5a27434d02..60c74154cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBucketsRatis.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBucketsRatis.java @@ -49,17 +49,17 @@ public static void shutdown() { } @Test - public void testCreateBucket() throws OzoneException, IOException { + public void testCreateBucket() throws Exception { TestBuckets.runTestCreateBucket(ozoneRestClient); } @Test - public void testAddBucketAcls() throws OzoneException, IOException { + public void testAddBucketAcls() throws Exception { TestBuckets.runTestAddBucketAcls(ozoneRestClient); } @Test - public void testRemoveBucketAcls() throws OzoneException, IOException { + public void testRemoveBucketAcls() throws Exception { TestBuckets.runTestRemoveBucketAcls(ozoneRestClient); } @@ -69,7 +69,7 @@ public void testDeleteBucket() throws OzoneException, IOException { } @Test - public void testListBucket() throws OzoneException, IOException { + public void testListBucket() throws Exception { TestBuckets.runTestListBucket(ozoneRestClient); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java index 69e9dcfba5..9c28f089f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java @@ -160,27 +160,29 @@ static void runTestDeleteVolume(OzoneRestClient client) } @Test - public void testChangeOwnerOnVolume() throws OzoneException { + public void testChangeOwnerOnVolume() throws Exception { runTestChangeOwnerOnVolume(ozoneRestClient); } static void runTestChangeOwnerOnVolume(OzoneRestClient client) - throws OzoneException { + throws OzoneException, ParseException { String volumeName = OzoneUtils.getRequestID().toLowerCase(); client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER); OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB"); client.setVolumeOwner(volumeName, "frodo"); OzoneVolume newVol = client.getVolume(volumeName); assertEquals(newVol.getOwnerName(), "frodo"); + // verify if the creation time is missing after setting owner operation + assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0); } @Test - public void testChangeQuotaOnVolume() throws OzoneException, IOException { + public void testChangeQuotaOnVolume() throws Exception { runTestChangeQuotaOnVolume(ozoneRestClient); } static void runTestChangeQuotaOnVolume(OzoneRestClient client) - throws OzoneException, IOException { + throws OzoneException, IOException, ParseException { String volumeName = OzoneUtils.getRequestID().toLowerCase(); client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER); OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB"); @@ -188,6 +190,8 @@ static void runTestChangeQuotaOnVolume(OzoneRestClient client) OzoneVolume newVol = client.getVolume(volumeName); assertEquals(newVol.getQuota().getSize(), 1000); assertEquals(newVol.getQuota().getUnit(), OzoneQuota.Units.MB); + // verify if the creation time is missing after setting quota operation + assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java index 23f322b63f..8704837cdb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java @@ -62,12 +62,12 @@ public void testDeleteVolume() throws OzoneException { } @Test - public void testChangeOwnerOnVolume() throws OzoneException { + public void testChangeOwnerOnVolume() throws Exception { TestVolume.runTestChangeOwnerOnVolume(ozoneClient); } @Test - public void testChangeQuotaOnVolume() throws OzoneException, IOException { + public void testChangeQuotaOnVolume() throws Exception { TestVolume.runTestChangeQuotaOnVolume(ozoneClient); }