big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12230. Ozone: KSM: Add creation time field in bucket info. Contributed by Yiqun Lin.

Browse Source
This commit is contained in:
Weiwei Yang 2017-08-15 14:17:33 +08:00 committed by Owen O'Malley
parent dcc21a4f93
commit 25586607a1
12 changed files with 117 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java
View File

@ -54,6 +54,10 @@ public final class KsmBucketInfo {
* [RAM_DISK, SSD, DISK, ARCHIVE]
*/
private StorageType storageType;
/**
* Creation time of bucket.
*/
private final long creationTime;
/**
* Private constructor, constructed via builder.
@ -62,15 +66,17 @@ public final class KsmBucketInfo {
* @param acls - list of ACLs.
* @param isVersionEnabled - Bucket version flag.
* @param storageType - Storage type to be used.
* @param creationTime - Bucket creation time.
*/
private KsmBucketInfo(String volumeName, String bucketName,
List<OzoneAcl> acls, boolean isVersionEnabled,
StorageType storageType) {
StorageType storageType, long creationTime) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.acls = acls;
this.isVersionEnabled = isVersionEnabled;
this.storageType = storageType;
this.creationTime = creationTime;
}
/**
@ -113,6 +119,15 @@ public StorageType getStorageType() {
return storageType;
}
/**
* Returns creation time.
*
* @return long
*/
public long getCreationTime() {
return creationTime;
}
/**
* Returns new builder class that builds a KsmBucketInfo.
*
@ -131,6 +146,7 @@ public static class Builder {
private List<OzoneAcl> acls;
private Boolean isVersionEnabled;
private StorageType storageType;
private long creationTime;
Builder() {
//Default values
@ -164,6 +180,11 @@ public Builder setStorageType(StorageType storage) {
return this;
}
public Builder setCreationTime(long createdOn) {
this.creationTime = createdOn;
return this;
}
/**
* Constructs the KsmBucketInfo.
* @return instance of KsmBucketInfo.
@ -174,8 +195,9 @@ public KsmBucketInfo build() {
Preconditions.checkNotNull(acls);
Preconditions.checkNotNull(isVersionEnabled);
Preconditions.checkNotNull(storageType);
return new KsmBucketInfo(volumeName, bucketName, acls,
isVersionEnabled, storageType);
isVersionEnabled, storageType, creationTime);
}
}
@ -191,6 +213,7 @@ public BucketInfo getProtobuf() {
.setIsVersionEnabled(isVersionEnabled)
.setStorageType(PBHelperClient.convertStorageType(
storageType))
.setCreationTime(creationTime)
.build();
}
@ -207,6 +230,6 @@ public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) {
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketInfo.getIsVersionEnabled(),
PBHelperClient.convertStorageType(
bucketInfo.getStorageType()));
bucketInfo.getStorageType()), bucketInfo.getCreationTime());
}
}

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
View File

@ -162,6 +162,7 @@ message BucketInfo {
repeated OzoneAclInfo acls = 3;
required bool isVersionEnabled = 4 [default = false];
required StorageTypeProto storageType = 5 [default = DISK];
required uint64 creationTime = 6;
}
message BucketArgs {

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
View File

@ -202,6 +202,7 @@ public void setBucketProperty(KsmBucketArgs args) throws IOException {
bucketInfoBuilder.setIsVersionEnabled(
oldBucketInfo.getIsVersionEnabled());
}
bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
metadataManager.put(bucketKey, bucketInfoBuilder.build()
.getProtobuf().toByteArray());

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
View File

@ -180,6 +180,7 @@ public void setOwner(String volume, String owner) throws IOException {
.setAdminName(volumeArgs.getAdminName())
.setOwnerName(owner)
.setQuotaInBytes(volumeArgs.getQuotaInBytes())
.setCreationTime(volumeArgs.getCreationTime())
.build();
VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
@ -221,6 +222,7 @@ public void setQuota(String volume, long quota) throws IOException {
.setAdminName(volumeArgs.getAdminName())
.setOwnerName(volumeArgs.getOwnerName())
.setQuotaInBytes(quota)
.setCreationTime(volumeArgs.getCreationTime())
.build();
VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
View File

@ -146,6 +146,15 @@ public StorageType getStorageType() {
return bucketInfo.getStorageType();
}
/**
* Gets the creation time of the bucket.
*
* @return String
*/
public String getCreatedOn() {
return bucketInfo.getCreatedOn();
}
/**
* Puts an Object in Ozone bucket.
*

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
View File

@ -62,6 +62,7 @@ public class BucketInfo implements Comparable<BucketInfo> {
private String volumeName;
private String bucketName;
private String createdOn;
private List<OzoneAcl> acls;
private OzoneConsts.Versioning versioning;
private StorageType storageType;
@ -173,6 +174,24 @@ public void setBucketName(String bucketName) {
this.bucketName = bucketName;
}
/**
* Sets creation time of the bucket.
*
* @param creationTime - Date String
*/
public void setCreatedOn(String creationTime) {
this.createdOn = creationTime;
}
/**
* Returns creation time.
*
* @return creation time of bucket.
*/
public String getCreatedOn() {
return createdOn;
}
/**
* Returns a JSON string of this object.
* After stripping out bytesUsed and keyCount

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -239,6 +239,7 @@ public void createBucket(final BucketArgs args)
builder.setIsVersionEnabled(getBucketVersioningProtobuf(
args.getVersioning()));
}
builder.setCreationTime(Time.now());
keySpaceManagerClient.createBucket(builder.build());
}
@ -349,6 +350,7 @@ public ListBuckets listBuckets(ListArgs args)
bk.setBucketName(bucketInfo.getBucketName());
bk.setStorageType(bucketInfo.getStorageType());
bk.setAcls(bucketInfo.getAcls());
bk.setCreatedOn(OzoneUtils.formatTime(bucketInfo.getCreationTime()));
result.addBucket(bk);
}
return result;
@ -375,6 +377,8 @@ public BucketInfo getBucketInfo(BucketArgs args)
}
bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
bucketInfo.setAcls(ksmBucketInfo.getAcls());
bucketInfo.setCreatedOn(
OzoneUtils.formatTime(ksmBucketInfo.getCreationTime()));
return bucketInfo;
}

13
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
View File

@ -408,7 +408,11 @@ public void testInfoBucket() throws Exception {
String[] args = new String[] {"-infoBucket",
url + "/" + vol.getVolumeName() + "/" + bucketName};
assertEquals(0, ToolRunner.run(shell, args));
assertTrue(out.toString().contains(bucketName));
String output = out.toString();
assertTrue(output.contains(bucketName));
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
// test get info from a non-exist bucket
args = new String[] {"-infoBucket",
@ -429,6 +433,9 @@ public void testUpdateBucket() throws Exception {
url + "/" + vol.getVolumeName() + "/" + bucketName, "-addAcl",
"user:frodo:rw,group:samwise:r"};
assertEquals(0, ToolRunner.run(shell, args));
String output = out.toString();
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
bucket = vol.getBucket(bucketName);
assertEquals(2, bucket.getAcls().size());
@ -480,8 +487,11 @@ public void testListBucket() throws Exception {
List<String> volumes = getValueLines("volumeName", out.toString());
List<String> buckets = getValueLines("bucketName", out.toString());
List<String> creationTimes = getValueLines("createdOn", out.toString());
assertEquals(11, volumes.size());
assertEquals(11, buckets.size());
assertEquals(11, creationTimes.size());
// sort bucket names since the return buckets isn't in created order
Collections.sort(bucketNames);
// return bucket names should be [test-bucket0, test-bucket1,
@ -489,6 +499,7 @@ public void testListBucket() throws Exception {
for (int i = 0; i < buckets.size(); i++) {
assertTrue(buckets.get(i).contains(bucketNames.get(i)));
assertTrue(volumes.get(i).contains(vol.getVolumeName()));
assertTrue(creationTimes.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
}
out.reset();

37
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
@ -35,10 +36,12 @@
import java.io.IOException;
import java.net.URISyntaxException;
import java.text.ParseException;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestBuckets {
@ -90,12 +93,12 @@ public static void shutdown() {
}
@Test
public void testCreateBucket() throws OzoneException, IOException {
public void testCreateBucket() throws Exception {
runTestCreateBucket(ozoneRestClient);
}
static void runTestCreateBucket(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -103,10 +106,15 @@ static void runTestCreateBucket(OzoneRestClient client)
// create 10 buckets under same volume
for (int x = 0; x < 10; x++) {
long currentTime = Time.now();
String bucketName = OzoneUtils.getRequestID().toLowerCase();
OzoneBucket bucket =
vol.createBucket(bucketName, acls, StorageType.DEFAULT);
assertEquals(bucket.getBucketName(), bucketName);
// verify the bucket creation time
assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn())
/ 1000) >= (currentTime / 1000));
}
client.close();
@ -118,12 +126,12 @@ static void runTestCreateBucket(OzoneRestClient client)
}
@Test
public void testAddBucketAcls() throws OzoneException, IOException {
public void testAddBucketAcls() throws Exception {
runTestAddBucketAcls(ozoneRestClient);
}
static void runTestAddBucketAcls(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -133,16 +141,19 @@ static void runTestAddBucketAcls(OzoneRestClient client)
vol.addAcls(bucketName, acls);
OzoneBucket updatedBucket = vol.getBucket(bucketName);
assertEquals(updatedBucket.getAcls().size(), 2);
// verify if the creation time is missing after update operation
assertTrue(
(OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0);
client.close();
}
@Test
public void testRemoveBucketAcls() throws OzoneException, IOException {
public void testRemoveBucketAcls() throws Exception {
runTestRemoveBucketAcls(ozoneRestClient);
}
static void runTestRemoveBucketAcls(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -155,6 +166,9 @@ static void runTestRemoveBucketAcls(OzoneRestClient client)
// We removed all acls
assertEquals(updatedBucket.getAcls().size(), 0);
// verify if the creation time is missing after update operation
assertTrue(
(OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0);
client.close();
}
@ -183,16 +197,18 @@ static void runTestDeleteBucket(OzoneRestClient client)
}
@Test
public void testListBucket() throws OzoneException, IOException {
public void testListBucket() throws Exception {
runTestListBucket(ozoneRestClient);
}
static void runTestListBucket(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
String[] acls = {"user:frodo:rw", "user:samwise:rw"};
long currentTime = Time.now();
for (int x = 0; x < 10; x++) {
String bucketName = "listbucket-test-" + x;
vol.createBucket(bucketName, acls);
@ -200,6 +216,11 @@ static void runTestListBucket(OzoneRestClient client)
List<OzoneBucket> bucketList = vol.listBuckets("100", null, null);
assertEquals(bucketList.size(), 10);
for (OzoneBucket bucket : bucketList) {
assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn())
/ 1000) >= (currentTime / 1000));
}
bucketList = vol.listBuckets("3", null, null);
assertEquals(bucketList.size(), 3);

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBucketsRatis.java
View File

@ -49,17 +49,17 @@ public static void shutdown() {
}
@Test
public void testCreateBucket() throws OzoneException, IOException {
public void testCreateBucket() throws Exception {
TestBuckets.runTestCreateBucket(ozoneRestClient);
}
@Test
public void testAddBucketAcls() throws OzoneException, IOException {
public void testAddBucketAcls() throws Exception {
TestBuckets.runTestAddBucketAcls(ozoneRestClient);
}
@Test
public void testRemoveBucketAcls() throws OzoneException, IOException {
public void testRemoveBucketAcls() throws Exception {
TestBuckets.runTestRemoveBucketAcls(ozoneRestClient);
}
@ -69,7 +69,7 @@ public void testDeleteBucket() throws OzoneException, IOException {
}
@Test
public void testListBucket() throws OzoneException, IOException {
public void testListBucket() throws Exception {
TestBuckets.runTestListBucket(ozoneRestClient);
}
}

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View File

@ -160,27 +160,29 @@ static void runTestDeleteVolume(OzoneRestClient client)
}
@Test
public void testChangeOwnerOnVolume() throws OzoneException {
public void testChangeOwnerOnVolume() throws Exception {
runTestChangeOwnerOnVolume(ozoneRestClient);
}
static void runTestChangeOwnerOnVolume(OzoneRestClient client)
throws OzoneException {
throws OzoneException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
client.setVolumeOwner(volumeName, "frodo");
OzoneVolume newVol = client.getVolume(volumeName);
assertEquals(newVol.getOwnerName(), "frodo");
// verify if the creation time is missing after setting owner operation
assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0);
}
@Test
public void testChangeQuotaOnVolume() throws OzoneException, IOException {
public void testChangeQuotaOnVolume() throws Exception {
runTestChangeQuotaOnVolume(ozoneRestClient);
}
static void runTestChangeQuotaOnVolume(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -188,6 +190,8 @@ static void runTestChangeQuotaOnVolume(OzoneRestClient client)
OzoneVolume newVol = client.getVolume(volumeName);
assertEquals(newVol.getQuota().getSize(), 1000);
assertEquals(newVol.getQuota().getUnit(), OzoneQuota.Units.MB);
// verify if the creation time is missing after setting quota operation
assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0);
}
@Test

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java
View File

@ -62,12 +62,12 @@ public void testDeleteVolume() throws OzoneException {
}
@Test
public void testChangeOwnerOnVolume() throws OzoneException {
public void testChangeOwnerOnVolume() throws Exception {
TestVolume.runTestChangeOwnerOnVolume(ozoneClient);
}
@Test
public void testChangeQuotaOnVolume() throws OzoneException, IOException {
public void testChangeQuotaOnVolume() throws Exception {
TestVolume.runTestChangeQuotaOnVolume(ozoneClient);
}