HDFS-12170. Ozone: OzoneFileSystem: KSM should maintain key creation time and modification time. Contributed by Mukul Kumar Singh.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.util;
 
+import java.text.ParseException;
 import java.text.SimpleDateFormat;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -82,4 +83,12 @@ public static long monotonicNowNanos() {
   public static String formatTime(long millis) {
     return DATE_FORMAT.get().format(millis);
   }
+
+  /**
+   * Convert time in human readable format to millisecond.
+   * @return time in milliseconds
+   */
+  public static long formatDate(String date) throws ParseException {
+    return DATE_FORMAT.get().parse(date).getTime();
+  }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java

@@ -35,10 +35,13 @@ public final class KsmKeyInfo {
   private final String blockID;
   private final long dataSize;
   private final boolean shouldCreateContainer;
+  private final long creationTime;
+  private final long modificationTime;
 
   private KsmKeyInfo(String volumeName, String bucketName, String keyName,
       long dataSize, String blockID, String containerName,
-      boolean shouldCreateContainer) {
+      boolean shouldCreateContainer, long creationTime,
+      long modificationTime) {
     this.volumeName = volumeName;
     this.bucketName = bucketName;
     this.keyName = keyName;
@@ -46,6 +49,8 @@ private KsmKeyInfo(String volumeName, String bucketName, String keyName,
     this.blockID = blockID;
     this.dataSize = dataSize;
     this.shouldCreateContainer = shouldCreateContainer;
+    this.creationTime = creationTime;
+    this.modificationTime = modificationTime;
   }
 
   public String getVolumeName() {
@@ -76,6 +81,14 @@ public boolean getShouldCreateContainer() {
     return shouldCreateContainer;
   }
 
+  public long getCreationTime() {
+    return creationTime;
+  }
+
+  public long getModificationTime() {
+    return modificationTime;
+  }
+
   /**
    * Builder of KsmKeyInfo.
    */
@@ -87,6 +100,8 @@ public static class Builder {
     private String blockID;
     private long dataSize;
     private boolean shouldCreateContainer;
+    private long creationTime;
+    private long modificationTime;
 
     public Builder setVolumeName(String volume) {
       this.volumeName = volume;
@@ -123,10 +138,20 @@ public Builder setShouldCreateContainer(boolean create) {
       return this;
     }
 
+    public Builder setCreationTime(long creationTime) {
+      this.creationTime = creationTime;
+      return this;
+    }
+
+    public Builder setModificationTime(long modificationTime) {
+      this.modificationTime = modificationTime;
+      return this;
+    }
+
     public KsmKeyInfo build() {
       return new KsmKeyInfo(
           volumeName, bucketName, keyName, dataSize, blockID, containerName,
-          shouldCreateContainer);
+          shouldCreateContainer, creationTime, modificationTime);
     }
   }
 
@@ -139,6 +164,8 @@ public KeyInfo getProtobuf() {
         .setBlockKey(blockID)
         .setContainerName(containerName)
         .setShouldCreateContainer(shouldCreateContainer)
+        .setCreationTime(creationTime)
+        .setModificationTime(modificationTime)
         .build();
   }
 
@@ -150,7 +177,9 @@ public static KsmKeyInfo getFromProtobuf(KeyInfo keyInfo) {
         keyInfo.getDataSize(),
         keyInfo.getBlockKey(),
         keyInfo.getContainerName(),
-        keyInfo.getShouldCreateContainer());
+        keyInfo.getShouldCreateContainer(),
+        keyInfo.getCreationTime(),
+        keyInfo.getModificationTime());
   }
 
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto

@@ -233,6 +233,8 @@ message KeyInfo {
     required string blockKey = 5;
     required string containerName = 6;
     required bool shouldCreateContainer = 7;
+    required uint64 creationTime = 8;
+    required uint64 modificationTime = 9;
 }
 
 message LocateKeyRequest {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java

@@ -26,6 +26,7 @@
 import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
 import org.apache.hadoop.scm.container.common.helpers.DeleteBlockResult;
 import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
+import org.apache.hadoop.util.Time;
 import org.iq80.leveldb.DBException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -95,6 +96,7 @@ public KsmKeyInfo allocateKey(KsmKeyArgs args) throws IOException {
       // metadata entry in case of 0 length key.
       AllocatedBlock allocatedBlock =
           scmBlockClient.allocateBlock(Math.max(args.getDataSize(), 1));
+      long currentTime = Time.now();
       KsmKeyInfo keyBlock = new KsmKeyInfo.Builder()
           .setVolumeName(args.getVolumeName())
           .setBucketName(args.getBucketName())
@@ -103,6 +105,8 @@ public KsmKeyInfo allocateKey(KsmKeyArgs args) throws IOException {
           .setBlockID(allocatedBlock.getKey())
           .setContainerName(allocatedBlock.getPipeline().getContainerName())
           .setShouldCreateContainer(allocatedBlock.getCreateContainer())
+          .setCreationTime(currentTime)
+          .setModificationTime(currentTime)
           .build();
       metadataManager.put(keyKey, keyBlock.getProtobuf().toByteArray());
       LOG.debug("Key {} allocated in volume {} bucket {}",

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java

@@ -73,7 +73,7 @@ public Response getKey(String volume, String bucket, String key, String info,
        * @param input - The body as an Input Stream
        * @param request - Http request
        * @param headers - Parsed http Headers.
-       * @param info - UriInfo
+       * @param uriInfo - UriInfo
        *
        * @return Response
        *

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java

@@ -69,6 +69,7 @@ class MixIn {
   private long version;
   private String md5hash;
   private String createdOn;
+  private String modifiedOn;
   private long size;
   private String keyName;
 
@@ -83,6 +84,15 @@ public String getCreatedOn() {
     return createdOn;
   }
 
+  /**
+   * When this key was modified.
+   *
+   * @return Date String
+   */
+  public String getModifiedOn() {
+    return modifiedOn;
+  }
+
   /**
    * When this key was created.
    *
@@ -92,6 +102,15 @@ public void setCreatedOn(String createdOn) {
     this.createdOn = createdOn;
   }
 
+  /**
+   * When this key was modified.
+   *
+   * @param modifiedOn - Date String
+   */
+  public void setModifiedOn(String modifiedOn) {
+    this.modifiedOn = modifiedOn;
+  }
+
   /**
    * Full path to where the actual data for this key is stored.
    *

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java

@@ -66,6 +66,7 @@
 import org.apache.hadoop.scm.storage.ChunkOutputStream;
 import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -482,7 +483,8 @@ public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
     keyInfo.setVersion(0);
     keyInfo.setKeyName(ksmKeyInfo.getKeyName());
     keyInfo.setSize(ksmKeyInfo.getDataSize());
-
+    keyInfo.setCreatedOn(Time.formatTime(ksmKeyInfo.getCreationTime()));
+    keyInfo.setModifiedOn(Time.formatTime(ksmKeyInfo.getModificationTime()));
     return keyInfo;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java

@@ -44,6 +44,7 @@
 import org.apache.hadoop.ozone.web.response.ListBuckets;
 import org.apache.hadoop.ozone.web.response.ListKeys;
 import org.apache.hadoop.ozone.web.response.ListVolumes;
+import org.apache.hadoop.util.Time;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -55,6 +56,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.text.ParseException;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.Random;
@@ -941,11 +943,13 @@ public void testListVolumes() throws IOException, OzoneException {
    * @throws OzoneException
    */
   @Test
-  public void testGetKeyInfo() throws IOException, OzoneException {
+  public void testGetKeyInfo() throws IOException,
+                          OzoneException, ParseException {
     String userName = "user" + RandomStringUtils.randomNumeric(5);
     String adminName = "admin" + RandomStringUtils.randomNumeric(5);
     String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
     String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+    long currentTime = Time.monotonicNow();
 
     VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
     createVolumeArgs.setUserName(userName);
@@ -966,6 +970,8 @@ public void testGetKeyInfo() throws IOException, OzoneException {
     stream.close();
 
     KeyInfo keyInfo = storageHandler.getKeyInfo(keyArgs);
+    Assert.assertTrue(Time.formatDate(keyInfo.getCreatedOn()) >= currentTime);
+    Assert.assertTrue(Time.formatDate(keyInfo.getModifiedOn()) >= currentTime);
     Assert.assertEquals(keyName, keyInfo.getKeyName());
     Assert.assertEquals(4096, keyInfo.getSize());
   }