From bb5f1c4228f4d6ff23e2cf6ba1c25408a8bc6165 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Fri, 4 Aug 2017 19:50:48 -0700 Subject: [PATCH] HDFS-12231. Ozone: KSM: Add creation time field in volume info. Contributed by Yiqun Lin. --- .../hadoop/ksm/helpers/KsmVolumeArgs.java | 25 +++++++++++++++--- .../main/proto/KeySpaceManagerProtocol.proto | 1 + .../hadoop/ozone/web/client/OzoneVolume.java | 9 +++++++ .../storage/DistributedStorageHandler.java | 8 ++++-- .../hadoop/ozone/ozShell/TestOzoneShell.java | 13 +++++++++- .../hadoop/ozone/web/client/TestVolume.java | 26 +++++++++++++------ .../ozone/web/client/TestVolumeRatis.java | 6 ++--- 7 files changed, 71 insertions(+), 17 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java index 7831a8752d..f7c6593d0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java @@ -39,6 +39,7 @@ public final class KsmVolumeArgs { private final String adminName; private final String ownerName; private final String volume; + private final long creationTime; private final long quotaInBytes; private final Map keyValueMap; private final KsmOzoneAclMap aclMap; @@ -51,16 +52,18 @@ public final class KsmVolumeArgs { * @param quotaInBytes - Volume Quota in bytes. * @param keyValueMap - keyValue map. * @param aclMap - User to access rights map. + * @param creationTime - Volume creation time. */ private KsmVolumeArgs(String adminName, String ownerName, String volume, long quotaInBytes, Map keyValueMap, - KsmOzoneAclMap aclMap) { + KsmOzoneAclMap aclMap, long creationTime) { this.adminName = adminName; this.ownerName = ownerName; this.volume = volume; this.quotaInBytes = quotaInBytes; this.keyValueMap = keyValueMap; this.aclMap = aclMap; + this.creationTime = creationTime; } /** @@ -87,6 +90,14 @@ public String getVolume() { return volume; } + /** + * Returns creation time. + * @return long + */ + public long getCreationTime() { + return creationTime; + } + /** * Returns Quota in Bytes. * @return long, Quota in bytes. @@ -118,6 +129,7 @@ public static class Builder { private String adminName; private String ownerName; private String volume; + private long creationTime; private long quotaInBytes; private Map keyValueMap; private KsmOzoneAclMap aclMap; @@ -145,6 +157,11 @@ public Builder setVolume(String volume) { return this; } + public Builder setCreationTime(long createdOn) { + this.creationTime = createdOn; + return this; + } + public Builder setQuotaInBytes(long quotaInBytes) { this.quotaInBytes = quotaInBytes; return this; @@ -169,7 +186,7 @@ public KsmVolumeArgs build() { Preconditions.checkNotNull(ownerName); Preconditions.checkNotNull(volume); return new KsmVolumeArgs(adminName, ownerName, volume, quotaInBytes, - keyValueMap, aclMap); + keyValueMap, aclMap, creationTime); } } @@ -188,6 +205,7 @@ public VolumeInfo getProtobuf() { .setQuotaInBytes(quotaInBytes) .addAllMetadata(metadataList) .addAllVolumeAcls(aclList) + .setCreationTime(creationTime) .build(); } @@ -199,6 +217,7 @@ public static KsmVolumeArgs getFromProtobuf(VolumeInfo volInfo) { KsmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList()); return new KsmVolumeArgs(volInfo.getAdminName(), volInfo.getOwnerName(), - volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap); + volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap, + volInfo.getCreationTime()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto index 956e7fad35..db364beaa8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto @@ -62,6 +62,7 @@ message VolumeInfo { optional uint64 quotaInBytes = 4; repeated KeyValue metadata = 5; repeated OzoneAclInfo volumeAcls = 6; + required uint64 creationTime = 7; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java index ce9fd948be..f7cc649e9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java @@ -131,6 +131,15 @@ public OzoneQuota getQuota() { return volumeInfo.getQuota(); } + /** + * Returns creation time of Volume. + * + * @return String + */ + public String getCreatedOn() { + return volumeInfo.getCreatedOn(); + } + /** * Returns a Http header from the Last Volume related call. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java index ccc71fa553..713a0851be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java @@ -42,6 +42,7 @@ import org.apache.hadoop.scm.XceiverClientManager; import org.apache.hadoop.scm.protocolPB .StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.util.Time; import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.handlers.BucketArgs; import org.apache.hadoop.ozone.web.handlers.KeyArgs; @@ -124,7 +125,8 @@ public void createVolume(VolumeArgs args) throws IOException, OzoneException { .setOwnerName(args.getUserName()) .setVolume(args.getVolumeName()) .setQuotaInBytes(quota) - .addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl)); + .addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl)) + .setCreationTime(Time.now()); if (args.getGroups() != null) { for (String group : args.getGroups()) { OzoneAcl groupAcl = @@ -183,7 +185,7 @@ public ListVolumes listVolumes(ListArgs args) args.getMaxKeys()); } - // TODO Add missing fields createdOn, createdBy, bucketCount and bytesUsed + // TODO Add missing fields createdBy, bucketCount and bytesUsed ListVolumes result = new ListVolumes(); for (KsmVolumeArgs volumeArgs : listResult) { VolumeInfo info = new VolumeInfo(); @@ -192,6 +194,7 @@ public ListVolumes listVolumes(ListArgs args) info.setOwner(new VolumeOwner(infoProto.getOwnerName())); info.setQuota(OzoneQuota.getOzoneQuota(infoProto.getQuotaInBytes())); info.setVolumeName(infoProto.getVolume()); + info.setCreatedOn(OzoneUtils.formatTime(infoProto.getCreationTime())); result.addVolume(info); } @@ -215,6 +218,7 @@ public VolumeInfo getVolumeInfo(VolumeArgs args) volumeArgs.getAdminName()); volInfo.setOwner(new VolumeOwner(volumeArgs.getOwnerName())); volInfo.setQuota(OzoneQuota.getOzoneQuota(volumeArgs.getQuotaInBytes())); + volInfo.setCreatedOn(OzoneUtils.formatTime(volumeArgs.getCreationTime())); return volInfo; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java index b8e353541b..a41fc00ac9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java @@ -191,7 +191,11 @@ public void testInfoVolume() throws Exception { String[] args = new String[] {"-infoVolume", url + "/" + volumeName, "-root"}; assertEquals(0, ToolRunner.run(shell, args)); - assertTrue(out.toString().contains(volumeName)); + + String output = out.toString(); + assertTrue(output.contains(volumeName)); + assertTrue(output.contains("createdOn") + && output.contains(OzoneConsts.OZONE_TIME_ZONE)); // get info for non-exist volume args = new String[] {"-infoVolume", url + "/invalid-volume", "-root"}; @@ -270,12 +274,19 @@ public void testListVolume() throws Exception { List names = getValueLines("name", out.toString()); List volumes = getValueLines("volumeName", out.toString()); + List creationTimes = getValueLines("createdOn", out.toString()); assertEquals(10, volumes.size()); assertEquals(10, names.size()); + assertEquals(10, creationTimes.size()); + for (String user : names) { assertTrue(user.contains(user1)); } + for (String time : creationTimes) { + assertTrue(time.contains(OzoneConsts.OZONE_TIME_ZONE)); + } + out.reset(); args = new String[] {"-listVolume", url + "/", "-user", user1, "-length", "2"}; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java index 3762bd5134..33002aef5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java @@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.web.request.OzoneQuota; import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.log4j.Level; @@ -41,10 +42,10 @@ import java.io.File; import java.io.IOException; +import java.text.ParseException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -99,15 +100,16 @@ public static void shutdown() { } @Test - public void testCreateVolume() throws OzoneException, IOException { + public void testCreateVolume() throws Exception { runTestCreateVolume(ozoneRestClient); } static void runTestCreateVolume(OzoneRestClient client) - throws OzoneException, IOException { + throws OzoneException, IOException, ParseException { String volumeName = OzoneUtils.getRequestID().toLowerCase(); client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER); + long currentTime = Time.now(); OzoneRestClient mockClient = Mockito.spy(client); List mockedClients = mockHttpClients(mockClient); OzoneVolume vol = mockClient.createVolume(volumeName, "bilbo", "100TB"); @@ -119,6 +121,10 @@ static void runTestCreateVolume(OzoneRestClient client) assertEquals(vol.getOwnerName(), "bilbo"); assertEquals(vol.getQuota().getUnit(), OzoneQuota.Units.TB); assertEquals(vol.getQuota().getSize(), 100); + + // verify the key creation time + assertTrue((OzoneUtils.formatDate(vol.getCreatedOn()) + / 1000) >= (currentTime / 1000)); } @Test @@ -270,16 +276,17 @@ static void runTestListAllVolumes(OzoneRestClient client) } @Test - public void testListVolumes() throws OzoneException, IOException { + public void testListVolumes() throws Exception { runTestListVolumes(ozoneRestClient); } static void runTestListVolumes(OzoneRestClient client) - throws OzoneException, IOException { + throws OzoneException, IOException, ParseException { final int volCount = 20; final String user1 = "test-user-a"; final String user2 = "test-user-b"; + long currentTime = Time.now(); client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER); // Create 20 volumes, 10 for user1 and another 10 for user2. for (int x = 0; x < volCount; x++) { @@ -303,9 +310,12 @@ static void runTestListVolumes(OzoneRestClient client) List volumeList = client.listVolumes(user1, null, 100, StringUtils.EMPTY); assertEquals(10, volumeList.size()); - volumeList.stream() - .filter(item -> item.getOwnerName().equals(user1)) - .collect(Collectors.toList()); + // verify the owner name and creation time of volume + for (OzoneVolume vol : volumeList) { + assertTrue(vol.getOwnerName().equals(user1)); + assertTrue((OzoneUtils.formatDate(vol.getCreatedOn()) + / 1000) >= (currentTime / 1000)); + } // test max key parameter of listing volumes volumeList = client.listVolumes(user1, null, 2, StringUtils.EMPTY); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java index 91e6d2d9c0..23f322b63f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java @@ -47,7 +47,7 @@ public static void shutdown() { } @Test - public void testCreateVolume() throws OzoneException, IOException { + public void testCreateVolume() throws Exception { TestVolume.runTestCreateVolume(ozoneClient); } @@ -88,12 +88,12 @@ public void testListVolumePagination() throws OzoneException, IOException { // TODO: remove @Ignore below once the problem has been resolved. @Ignore("See TestVolume.testListAllVolumes()") @Test - public void testListAllVolumes() throws OzoneException, IOException { + public void testListAllVolumes() throws Exception { TestVolume.runTestListAllVolumes(ozoneClient); } @Test - public void testListVolumes() throws OzoneException, IOException { + public void testListVolumes() throws Exception { TestVolume.runTestListVolumes(ozoneClient); } }