big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12490. Ozone: OzoneClient: Add creation/modification time information in OzoneVolume/OzoneBucket/OzoneKey. Contributed by Mukul Kumar Singh.

Browse Source
This commit is contained in:
Nanda kumar 2017-10-12 21:52:36 +05:30 committed by Owen O'Malley
parent 8881309002
commit 9bb516118d
10 changed files with 149 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
View File

@ -72,6 +72,11 @@ public class OzoneBucket {
*/
private int listCacheSize;
/**
* Creation time of the bucket.
*/
private long creationTime;
/**
* Constructs OzoneBucket instance.
* @param conf Configuration object.
@ -81,11 +86,12 @@ public class OzoneBucket {
* @param acls ACLs associated with the bucket.
* @param storageType StorageType of the bucket.
* @param versioning versioning status of the bucket.
* @param creationTime creation time of the bucket.
*/
public OzoneBucket(Configuration conf, ClientProtocol proxy,
String volumeName, String bucketName,
List<OzoneAcl> acls, StorageType storageType,
Boolean versioning) {
Boolean versioning, long creationTime) {
this.proxy = proxy;
this.volumeName = volumeName;
this.name = bucketName;
@ -93,6 +99,7 @@ public OzoneBucket(Configuration conf, ClientProtocol proxy,
this.storageType = storageType;
this.versioning = versioning;
this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
this.creationTime = creationTime;
}
/**
@ -140,6 +147,15 @@ public Boolean getVersioning() {
return versioning;
}
/**
* Returns creation time of the Bucket.
*
* @return creation time of the bucket
*/
public long getCreationTime() {
return creationTime;
}
/**
* Adds ACLs to the Bucket.
* @param addAcls ACLs to be added

30
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
View File

@ -39,17 +39,28 @@ public class OzoneKey {
* Size of the data.
*/
private final long dataSize;
/**
* Creation time of the key.
*/
private long creationTime;
/**
* Modification time of the key.
*/
private long modificationTime;
/**
* Constructs OzoneKey from KsmKeyInfo.
*
*/
public OzoneKey(String volumeName, String bucketName,
String keyName, long size) {
String keyName, long size, long creationTime,
long modificationTime) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.name = keyName;
this.dataSize = size;
this.creationTime = creationTime;
this.modificationTime = modificationTime;
}
/**
@ -88,4 +99,21 @@ public long getDataSize() {
return dataSize;
}
/**
* Returns the creation time of the key.
*
* @return creation time
*/
public long getCreationTime() {
return creationTime;
}
/**
* Returns the modification time of the key.
*
* @return modification time
*/
public long getModificationTime() {
return modificationTime;
}
}

17
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
View File

@ -56,6 +56,10 @@ public class OzoneVolume {
* Quota allocated for the Volume.
*/
private long quotaInBytes;
/**
* Creation time of the volume.
*/
private long creationTime;
/**
* Volume ACLs.
*/
@ -71,16 +75,18 @@ public class OzoneVolume {
* @param admin Volume admin.
* @param owner Volume owner.
* @param quotaInBytes Volume quota in bytes.
* @param creationTime creation time of the volume
* @param acls ACLs associated with the volume.
*/
public OzoneVolume(Configuration conf, ClientProtocol proxy, String name,
String admin, String owner, long quotaInBytes,
List<OzoneAcl> acls) {
long creationTime, List<OzoneAcl> acls) {
this.proxy = proxy;
this.name = name;
this.admin = admin;
this.owner = owner;
this.quotaInBytes = quotaInBytes;
this.creationTime = creationTime;
this.acls = acls;
this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
}
@ -121,6 +127,15 @@ public long getQuota() {
return quotaInBytes;
}
/**
* Returns creation time of the volume.
*
* @return creation time.
*/
public long getCreationTime() {
return creationTime;
}
/**
* Returns OzoneAcl list associated with the Volume.
*

17
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
View File

@ -226,6 +226,7 @@ public OzoneVolume getVolumeDetails(String volumeName)
volume.getAdminName(),
volume.getOwnerName(),
volume.getQuotaInBytes(),
volume.getCreationTime(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
@ -256,6 +257,7 @@ public List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume,
volume.getAdminName(),
volume.getOwnerName(),
volume.getQuotaInBytes(),
volume.getCreationTime(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
.collect(Collectors.toList());
@ -275,6 +277,7 @@ public List<OzoneVolume> listVolumes(String user, String volumePrefix,
volume.getAdminName(),
volume.getOwnerName(),
volume.getQuotaInBytes(),
volume.getCreationTime(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
.collect(Collectors.toList());
@ -409,7 +412,8 @@ public OzoneBucket getBucketDetails(
bucketArgs.getBucketName(),
bucketArgs.getAcls(),
bucketArgs.getStorageType(),
bucketArgs.getIsVersionEnabled());
bucketArgs.getIsVersionEnabled(),
bucketArgs.getCreationTime());
}
@Override
@ -426,7 +430,8 @@ public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
bucket.getBucketName(),
bucket.getAcls(),
bucket.getStorageType(),
bucket.getIsVersionEnabled()))
bucket.getIsVersionEnabled(),
bucket.getCreationTime()))
.collect(Collectors.toList());
}
@ -507,7 +512,9 @@ public List<OzoneKey> listKeys(String volumeName, String bucketName,
key.getVolumeName(),
key.getBucketName(),
key.getKeyName(),
key.getDataSize()))
key.getDataSize(),
key.getCreationTime(),
key.getModificationTime()))
.collect(Collectors.toList());
}
@ -527,7 +534,9 @@ public OzoneKey getKeyDetails(
return new OzoneKey(keyInfo.getVolumeName(),
keyInfo.getBucketName(),
keyInfo.getKeyName(),
keyInfo.getDataSize());
keyInfo.getDataSize(),
keyInfo.getCreationTime(),
keyInfo.getModificationTime());
}
@Override

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
View File

@ -35,7 +35,7 @@ public final class KsmKeyInfo {
private long dataSize;
private List<KsmKeyLocationInfo> keyLocationList;
private final long creationTime;
private final long modificationTime;
private long modificationTime;
private KsmKeyInfo(String volumeName, String bucketName, String keyName,
List<KsmKeyLocationInfo> locationInfos, long dataSize, long creationTime,
@ -85,6 +85,10 @@ public long getModificationTime() {
return modificationTime;
}
public void setModificationTime(long modificationTime) {
this.modificationTime = modificationTime;
}
/**
* Builder of KsmKeyInfo.
*/

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.util.Time;
import org.iq80.leveldb.DBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -96,7 +97,16 @@ public void createBucket(KsmBucketInfo bucketInfo) throws IOException {
throw new KSMException("Bucket already exist",
KSMException.ResultCodes.FAILED_BUCKET_ALREADY_EXISTS);
}
metadataManager.put(bucketKey, bucketInfo.getProtobuf().toByteArray());
KsmBucketInfo ksmBucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName(bucketInfo.getVolumeName())
.setBucketName(bucketInfo.getBucketName())
.setAcls(bucketInfo.getAcls())
.setStorageType(bucketInfo.getStorageType())
.setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
.setCreationTime(Time.now())
.build();
metadataManager.put(bucketKey, ksmBucketInfo.getProtobuf().toByteArray());
LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
} catch (IOException | DBException ex) {

40
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
View File

@ -27,7 +27,8 @@
import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException.ResultCodes;
import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.util.Time;
@ -43,18 +44,30 @@
import java.util.Random;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
import org.apache.hadoop.ozone.protocol
.proto.OzoneProtos.ReplicationType;
import org.apache.hadoop.ozone.protocol
.proto.OzoneProtos.ReplicationFactor;
/**
@ -296,6 +309,7 @@ public void commitKey(KsmKeyArgs args, int clientID) throws IOException {
KsmKeyInfo keyInfo =
KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(openKeyData));
keyInfo.setDataSize(args.getDataSize());
keyInfo.setModificationTime(Time.now());
BatchOperation batch = new BatchOperation();
batch.delete(openKey);
batch.put(objectKeyBytes, keyInfo.getProtobuf().toByteArray());

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
View File

@ -26,6 +26,8 @@
.KeySpaceManagerProtocolProtos.VolumeList;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.VolumeInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.BatchOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -33,9 +35,12 @@
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_USER_MAX_VOLUME_DEFAULT;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_USER_MAX_VOLUME;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_USER_MAX_VOLUME_DEFAULT;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_USER_MAX_VOLUME;
import static org.apache.hadoop.ozone.ksm.exceptions
.KSMException.ResultCodes;
@ -132,7 +137,22 @@ public void createVolume(KsmVolumeArgs args) throws IOException {
BatchOperation batch = new BatchOperation();
// Write the vol info
VolumeInfo newVolumeInfo = args.getProtobuf();
List<OzoneProtos.KeyValue> metadataList = new LinkedList<>();
for (Map.Entry<String, String> entry : args.getKeyValueMap().entrySet()) {
metadataList.add(OzoneProtos.KeyValue.newBuilder()
.setKey(entry.getKey()).setValue(entry.getValue()).build());
}
List<OzoneAclInfo> aclList = args.getAclMap().ozoneAclGetProtobuf();
VolumeInfo newVolumeInfo = VolumeInfo.newBuilder()
.setAdminName(args.getAdminName())
.setOwnerName(args.getOwnerName())
.setVolume(args.getVolume())
.setQuotaInBytes(args.getQuotaInBytes())
.addAllMetadata(metadataList)
.addAllVolumeAcls(aclList)
.setCreationTime(Time.now())
.build();
batch.put(dbVolumeKey, newVolumeInfo.toByteArray());
// Add volume to user list

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -45,7 +45,6 @@
import org.apache.hadoop.scm.XceiverClientManager;
import org.apache.hadoop.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
@ -142,8 +141,7 @@ public void createVolume(VolumeArgs args) throws IOException, OzoneException {
.setOwnerName(args.getUserName())
.setVolume(args.getVolumeName())
.setQuotaInBytes(quota)
.addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl))
.setCreationTime(Time.now());
.addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl));
if (args.getGroups() != null) {
for (String group : args.getGroups()) {
OzoneAcl groupAcl =
@ -255,7 +253,6 @@ public void createBucket(final BucketArgs args)
builder.setIsVersionEnabled(getBucketVersioningProtobuf(
args.getVersioning()));
}
builder.setCreationTime(Time.now());
keySpaceManagerClient.createBucket(builder.build());
}

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
View File

@ -38,6 +38,7 @@
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -177,6 +178,7 @@ public void testDeleteVolume()
@Test
public void testCreateBucket()
throws IOException, OzoneException {
long currentTime = Time.now();
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
store.createVolume(volumeName);
@ -184,6 +186,8 @@ public void testCreateBucket()
volume.createBucket(bucketName);
OzoneBucket bucket = volume.getBucket(bucketName);
Assert.assertEquals(bucketName, bucket.getName());
Assert.assertTrue(bucket.getCreationTime() >= currentTime);
Assert.assertTrue(volume.getCreationTime() >= currentTime);
}
@Test
@ -361,6 +365,7 @@ public void testPutKey()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
long currentTime = Time.now();
String value = "sample value";
store.createVolume(volumeName);
@ -382,6 +387,8 @@ public void testPutKey()
byte[] fileContent = new byte[value.getBytes().length];
is.read(fileContent);
Assert.assertEquals(value, new String(fileContent));
Assert.assertTrue(key.getCreationTime() >= currentTime);
Assert.assertTrue(key.getModificationTime() >= currentTime);
}
}