big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12170. Ozone: OzoneFileSystem: KSM should maintain key creation time and modification time. Contributed by Mukul Kumar Singh.

Browse Source
This commit is contained in:
Yiqun Lin 2017-07-27 11:43:01 +08:00 committed by Owen O'Malley
parent 349a87ac9c
commit 8cf0c864c4
8 changed files with 77 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.util;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import org.apache.hadoop.classification.InterfaceAudience;
@ -82,4 +83,12 @@ public static long monotonicNowNanos() {
public static String formatTime(long millis) {
return DATE_FORMAT.get().format(millis);
}
/**
* Convert time in human readable format to millisecond.
* @return time in milliseconds
*/
public static long formatDate(String date) throws ParseException {
return DATE_FORMAT.get().parse(date).getTime();
}
}

35
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java
View File

@ -35,10 +35,13 @@ public final class KsmKeyInfo {
private final String blockID;
private final long dataSize;
private final boolean shouldCreateContainer;
private final long creationTime;
private final long modificationTime;
private KsmKeyInfo(String volumeName, String bucketName, String keyName,
long dataSize, String blockID, String containerName,
boolean shouldCreateContainer) {
boolean shouldCreateContainer, long creationTime,
long modificationTime) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.keyName = keyName;
@ -46,6 +49,8 @@ private KsmKeyInfo(String volumeName, String bucketName, String keyName,
this.blockID = blockID;
this.dataSize = dataSize;
this.shouldCreateContainer = shouldCreateContainer;
this.creationTime = creationTime;
this.modificationTime = modificationTime;
}
public String getVolumeName() {
@ -76,6 +81,14 @@ public boolean getShouldCreateContainer() {
return shouldCreateContainer;
}
public long getCreationTime() {
return creationTime;
}
public long getModificationTime() {
return modificationTime;
}
/**
* Builder of KsmKeyInfo.
*/
@ -87,6 +100,8 @@ public static class Builder {
private String blockID;
private long dataSize;
private boolean shouldCreateContainer;
private long creationTime;
private long modificationTime;
public Builder setVolumeName(String volume) {
this.volumeName = volume;
@ -123,10 +138,20 @@ public Builder setShouldCreateContainer(boolean create) {
return this;
}
public Builder setCreationTime(long creationTime) {
this.creationTime = creationTime;
return this;
}
public Builder setModificationTime(long modificationTime) {
this.modificationTime = modificationTime;
return this;
}
public KsmKeyInfo build() {
return new KsmKeyInfo(
volumeName, bucketName, keyName, dataSize, blockID, containerName,
shouldCreateContainer);
shouldCreateContainer, creationTime, modificationTime);
}
}
@ -139,6 +164,8 @@ public KeyInfo getProtobuf() {
.setBlockKey(blockID)
.setContainerName(containerName)
.setShouldCreateContainer(shouldCreateContainer)
.setCreationTime(creationTime)
.setModificationTime(modificationTime)
.build();
}
@ -150,7 +177,9 @@ public static KsmKeyInfo getFromProtobuf(KeyInfo keyInfo) {
keyInfo.getDataSize(),
keyInfo.getBlockKey(),
keyInfo.getContainerName(),
keyInfo.getShouldCreateContainer());
keyInfo.getShouldCreateContainer(),
keyInfo.getCreationTime(),
keyInfo.getModificationTime());
}
}

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
View File

@ -233,6 +233,8 @@ message KeyInfo {
required string blockKey = 5;
required string containerName = 6;
required bool shouldCreateContainer = 7;
required uint64 creationTime = 8;
required uint64 modificationTime = 9;
}
message LocateKeyRequest {

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
View File

@ -26,6 +26,7 @@
import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.scm.container.common.helpers.DeleteBlockResult;
import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.util.Time;
import org.iq80.leveldb.DBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -95,6 +96,7 @@ public KsmKeyInfo allocateKey(KsmKeyArgs args) throws IOException {
// metadata entry in case of 0 length key.
AllocatedBlock allocatedBlock =
scmBlockClient.allocateBlock(Math.max(args.getDataSize(), 1));
long currentTime = Time.now();
KsmKeyInfo keyBlock = new KsmKeyInfo.Builder()
.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
@ -103,6 +105,8 @@ public KsmKeyInfo allocateKey(KsmKeyArgs args) throws IOException {
.setBlockID(allocatedBlock.getKey())
.setContainerName(allocatedBlock.getPipeline().getContainerName())
.setShouldCreateContainer(allocatedBlock.getCreateContainer())
.setCreationTime(currentTime)
.setModificationTime(currentTime)
.build();
metadataManager.put(keyKey, keyBlock.getProtobuf().toByteArray());
LOG.debug("Key {} allocated in volume {} bucket {}",

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
View File

@ -73,7 +73,7 @@ public Response getKey(String volume, String bucket, String key, String info,
* @param input - The body as an Input Stream
* @param request - Http request
* @param headers - Parsed http Headers.
* @param info - UriInfo
* @param uriInfo - UriInfo
*
* @return Response
*

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
View File

@ -69,6 +69,7 @@ class MixIn {
private long version;
private String md5hash;
private String createdOn;
private String modifiedOn;
private long size;
private String keyName;
@ -83,6 +84,15 @@ public String getCreatedOn() {
return createdOn;
}
/**
* When this key was modified.
*
* @return Date String
*/
public String getModifiedOn() {
return modifiedOn;
}
/**
* When this key was created.
*
@ -92,6 +102,15 @@ public void setCreatedOn(String createdOn) {
this.createdOn = createdOn;
}
/**
* When this key was modified.
*
* @param modifiedOn - Date String
*/
public void setModifiedOn(String modifiedOn) {
this.modifiedOn = modifiedOn;
}
/**
* Full path to where the actual data for this key is stored.
*

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -66,6 +66,7 @@
import org.apache.hadoop.scm.storage.ChunkOutputStream;
import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -482,7 +483,8 @@ public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
keyInfo.setVersion(0);
keyInfo.setKeyName(ksmKeyInfo.getKeyName());
keyInfo.setSize(ksmKeyInfo.getDataSize());
keyInfo.setCreatedOn(Time.formatTime(ksmKeyInfo.getCreationTime()));
keyInfo.setModifiedOn(Time.formatTime(ksmKeyInfo.getModificationTime()));
return keyInfo;
}

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
View File

@ -44,6 +44,7 @@
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.response.ListVolumes;
import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -55,6 +56,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.ParseException;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Random;
@ -941,11 +943,13 @@ public void testListVolumes() throws IOException, OzoneException {
* @throws OzoneException
*/
@Test
public void testGetKeyInfo() throws IOException, OzoneException {
public void testGetKeyInfo() throws IOException,
OzoneException, ParseException {
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
long currentTime = Time.monotonicNow();
VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
createVolumeArgs.setUserName(userName);
@ -966,6 +970,8 @@ public void testGetKeyInfo() throws IOException, OzoneException {
stream.close();
KeyInfo keyInfo = storageHandler.getKeyInfo(keyArgs);
Assert.assertTrue(Time.formatDate(keyInfo.getCreatedOn()) >= currentTime);
Assert.assertTrue(Time.formatDate(keyInfo.getModifiedOn()) >= currentTime);
Assert.assertEquals(keyName, keyInfo.getKeyName());
Assert.assertEquals(4096, keyInfo.getSize());
}