HDDS-2170. Add Object IDs and Update ID to Volume Object (#1510)

This commit is contained in:
Anu Engineer 2019-09-24 11:01:05 -07:00 committed by GitHub
parent afa1006a53
commit f16cf877e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 105 additions and 22 deletions

View File

@ -239,6 +239,8 @@ private OzoneConsts() {
public static final String KEY = "key"; public static final String KEY = "key";
public static final String QUOTA = "quota"; public static final String QUOTA = "quota";
public static final String QUOTA_IN_BYTES = "quotaInBytes"; public static final String QUOTA_IN_BYTES = "quotaInBytes";
public static final String OBJECT_ID = "objectID";
public static final String UPDATE_ID = "updateID";
public static final String CLIENT_ID = "clientID"; public static final String CLIENT_ID = "clientID";
public static final String OWNER = "owner"; public static final String OWNER = "owner";
public static final String ADMIN = "admin"; public static final String ADMIN = "admin";

View File

@ -44,6 +44,50 @@ public final class OmVolumeArgs extends WithMetadata implements Auditable {
private long creationTime; private long creationTime;
private long quotaInBytes; private long quotaInBytes;
private final OmOzoneAclMap aclMap; private final OmOzoneAclMap aclMap;
private long objectID;
private long updateID;
/**
* Set the Object ID. If this value is already set then this function throws.
* There is a reason why we cannot use the final here. The OmVolumeArgs is
* deserialized from the protobuf in many places in code. We need to set
* this object ID, after it is deserialized.
*
* @param obId - long
*/
public void setObjectID(long obId) {
if(this.objectID != 0) {
throw new UnsupportedOperationException("Attempt to modify object ID " +
"which is not zero. Current Object ID is " + this.objectID);
}
this.objectID = obId;
}
/**
* Returns a monotonically increasing ID, that denotes the last update.
* Each time an update happens, this ID is incremented.
* @return long
*/
public long getUpdateID() {
return updateID;
}
/**
* Sets the update ID. For each modification of this object, we will set
* this to a value greater than the current value.
* @param updateID long
*/
public void setUpdateID(long updateID) {
this.updateID = updateID;
}
/**
* A immutable identity field for this object.
* @return long.
*/
public long getObjectID() {
return objectID;
}
/** /**
* Private constructor, constructed via builder. * Private constructor, constructed via builder.
@ -54,10 +98,16 @@ public final class OmVolumeArgs extends WithMetadata implements Auditable {
* @param metadata - metadata map for custom key/value data. * @param metadata - metadata map for custom key/value data.
* @param aclMap - User to access rights map. * @param aclMap - User to access rights map.
* @param creationTime - Volume creation time. * @param creationTime - Volume creation time.
* @param objectID - ID of this object.
* @param updateID - A sequence number that denotes the last update on this
* object. This is a monotonically increasing number.
*/ */
@SuppressWarnings({"checkstyle:ParameterNumber", "This is invoked from a " +
"builder."})
private OmVolumeArgs(String adminName, String ownerName, String volume, private OmVolumeArgs(String adminName, String ownerName, String volume,
long quotaInBytes, Map<String, String> metadata, long quotaInBytes, Map<String, String> metadata,
OmOzoneAclMap aclMap, long creationTime) { OmOzoneAclMap aclMap, long creationTime, long objectID,
long updateID) {
this.adminName = adminName; this.adminName = adminName;
this.ownerName = ownerName; this.ownerName = ownerName;
this.volume = volume; this.volume = volume;
@ -65,6 +115,8 @@ private OmVolumeArgs(String adminName, String ownerName, String volume,
this.metadata = metadata; this.metadata = metadata;
this.aclMap = aclMap; this.aclMap = aclMap;
this.creationTime = creationTime; this.creationTime = creationTime;
this.objectID = objectID;
this.updateID = updateID;
} }
@ -152,6 +204,8 @@ public Map<String, String> toAuditMap() {
auditMap.put(OzoneConsts.VOLUME, this.volume); auditMap.put(OzoneConsts.VOLUME, this.volume);
auditMap.put(OzoneConsts.CREATION_TIME, String.valueOf(this.creationTime)); auditMap.put(OzoneConsts.CREATION_TIME, String.valueOf(this.creationTime));
auditMap.put(OzoneConsts.QUOTA_IN_BYTES, String.valueOf(this.quotaInBytes)); auditMap.put(OzoneConsts.QUOTA_IN_BYTES, String.valueOf(this.quotaInBytes));
auditMap.put(OzoneConsts.OBJECT_ID, String.valueOf(this.getObjectID()));
auditMap.put(OzoneConsts.UPDATE_ID, String.valueOf(this.getUpdateID()));
return auditMap; return auditMap;
} }
@ -164,17 +218,12 @@ public boolean equals(Object o) {
return false; return false;
} }
OmVolumeArgs that = (OmVolumeArgs) o; OmVolumeArgs that = (OmVolumeArgs) o;
return creationTime == that.creationTime && return Objects.equals(this.objectID, that.objectID);
quotaInBytes == that.quotaInBytes &&
Objects.equals(adminName, that.adminName) &&
Objects.equals(ownerName, that.ownerName) &&
Objects.equals(volume, that.volume);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(adminName, ownerName, volume, creationTime, return Objects.hash(this.objectID);
quotaInBytes);
} }
/** /**
@ -188,6 +237,29 @@ public static class Builder {
private long quotaInBytes; private long quotaInBytes;
private Map<String, String> metadata; private Map<String, String> metadata;
private OmOzoneAclMap aclMap; private OmOzoneAclMap aclMap;
private long objectID;
private long updateID;
/**
* Sets the Object ID for this Object.
* Object ID are unique and immutable identifier for each object in the
* System.
* @param objectID - long
*/
public void setObjectID(long objectID) {
this.objectID = objectID;
}
/**
* Sets the update ID for this Object. Update IDs are monotonically
* increasing values which are updated each time there is an update.
* @param updateID - long
*/
public void setUpdateID(long updateID) {
this.updateID = updateID;
}
/** /**
* Constructs a builder. * Constructs a builder.
@ -248,15 +320,13 @@ public OmVolumeArgs build() {
Preconditions.checkNotNull(ownerName); Preconditions.checkNotNull(ownerName);
Preconditions.checkNotNull(volume); Preconditions.checkNotNull(volume);
return new OmVolumeArgs(adminName, ownerName, volume, quotaInBytes, return new OmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
metadata, aclMap, creationTime); metadata, aclMap, creationTime, objectID, updateID);
} }
} }
public VolumeInfo getProtobuf() { public VolumeInfo getProtobuf() {
List<OzoneAclInfo> aclList = aclMap.ozoneAclGetProtobuf(); List<OzoneAclInfo> aclList = aclMap.ozoneAclGetProtobuf();
return VolumeInfo.newBuilder() return VolumeInfo.newBuilder()
.setAdminName(adminName) .setAdminName(adminName)
.setOwnerName(ownerName) .setOwnerName(ownerName)
@ -266,15 +336,15 @@ public VolumeInfo getProtobuf() {
.addAllVolumeAcls(aclList) .addAllVolumeAcls(aclList)
.setCreationTime( .setCreationTime(
creationTime == 0 ? System.currentTimeMillis() : creationTime) creationTime == 0 ? System.currentTimeMillis() : creationTime)
.setObjectID(objectID)
.setUpdateID(updateID)
.build(); .build();
} }
public static OmVolumeArgs getFromProtobuf(VolumeInfo volInfo) public static OmVolumeArgs getFromProtobuf(VolumeInfo volInfo)
throws OMException { throws OMException {
OmOzoneAclMap aclMap = OmOzoneAclMap aclMap =
OmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList()); OmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList());
return new OmVolumeArgs( return new OmVolumeArgs(
volInfo.getAdminName(), volInfo.getAdminName(),
volInfo.getOwnerName(), volInfo.getOwnerName(),
@ -282,6 +352,8 @@ public static OmVolumeArgs getFromProtobuf(VolumeInfo volInfo)
volInfo.getQuotaInBytes(), volInfo.getQuotaInBytes(),
KeyValueUtil.getFromProtobuf(volInfo.getMetadataList()), KeyValueUtil.getFromProtobuf(volInfo.getMetadataList()),
aclMap, aclMap,
volInfo.getCreationTime()); volInfo.getCreationTime(),
volInfo.getObjectID(),
volInfo.getUpdateID());
} }
} }

View File

@ -308,6 +308,8 @@ message VolumeInfo {
repeated hadoop.hdds.KeyValue metadata = 5; repeated hadoop.hdds.KeyValue metadata = 5;
repeated OzoneAclInfo volumeAcls = 6; repeated OzoneAclInfo volumeAcls = 6;
optional uint64 creationTime = 7; optional uint64 creationTime = 7;
optional uint64 objectID = 8;
optional uint64 updateID = 9;
} }
/** /**
@ -596,7 +598,6 @@ message InfoBucketRequest {
} }
message InfoBucketResponse { message InfoBucketResponse {
optional BucketInfo bucketInfo = 2; optional BucketInfo bucketInfo = 2;
} }
@ -755,7 +756,6 @@ message CreateKeyRequest {
} }
message CreateKeyResponse { message CreateKeyResponse {
optional KeyInfo keyInfo = 2; optional KeyInfo keyInfo = 2;
// clients' followup request may carry this ID for stateful operations // clients' followup request may carry this ID for stateful operations
// (similar to a cookie). // (similar to a cookie).
@ -768,7 +768,6 @@ message LookupKeyRequest {
} }
message LookupKeyResponse { message LookupKeyResponse {
optional KeyInfo keyInfo = 2; optional KeyInfo keyInfo = 2;
// clients' followup request may carry this ID for stateful operations (similar // clients' followup request may carry this ID for stateful operations (similar
// to a cookie). // to a cookie).
@ -844,7 +843,6 @@ message ListKeysRequest {
} }
message ListKeysResponse { message ListKeysResponse {
repeated KeyInfo keyInfo = 2; repeated KeyInfo keyInfo = 2;
} }
@ -953,7 +951,6 @@ message S3ListBucketsRequest {
} }
message S3ListBucketsResponse { message S3ListBucketsResponse {
repeated BucketInfo bucketInfo = 2; repeated BucketInfo bucketInfo = 2;
} }
@ -1024,7 +1021,6 @@ message MultipartUploadListPartsRequest {
} }
message MultipartUploadListPartsResponse { message MultipartUploadListPartsResponse {
optional hadoop.hdds.ReplicationType type = 2; optional hadoop.hdds.ReplicationType type = 2;
optional hadoop.hdds.ReplicationFactor factor = 3; optional hadoop.hdds.ReplicationFactor factor = 3;
optional uint32 nextPartNumberMarker = 4; optional uint32 nextPartNumberMarker = 4;

View File

@ -116,6 +116,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
Collection<String> ozAdmins = ozoneManager.getOzoneAdmins(); Collection<String> ozAdmins = ozoneManager.getOzoneAdmins();
try { try {
omVolumeArgs = OmVolumeArgs.getFromProtobuf(volumeInfo); omVolumeArgs = OmVolumeArgs.getFromProtobuf(volumeInfo);
// when you create a volume, we set both Object ID and update ID to the
// same ratis transaction ID. The Object ID will never change, but update
// ID will be set to transactionID each time we update the object.
omVolumeArgs.setUpdateID(transactionLogIndex);
omVolumeArgs.setObjectID(transactionLogIndex);
auditMap = omVolumeArgs.toAuditMap(); auditMap = omVolumeArgs.toAuditMap();
// check Acl // check Acl

View File

@ -20,6 +20,7 @@
import java.util.UUID; import java.util.UUID;
import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -73,6 +74,11 @@ public void testValidateAndUpdateCacheWithZeroMaxUserVolumeCount()
OMClientResponse omClientResponse = OMClientResponse omClientResponse =
omVolumeCreateRequest.validateAndUpdateCache(ozoneManager, 1, omVolumeCreateRequest.validateAndUpdateCache(ozoneManager, 1,
ozoneManagerDoubleBufferHelper); ozoneManagerDoubleBufferHelper);
Assert.assertTrue(omClientResponse instanceof OMVolumeCreateResponse);
OMVolumeCreateResponse respone =
(OMVolumeCreateResponse) omClientResponse;
Assert.assertEquals(1, respone.getOmVolumeArgs().getObjectID());
Assert.assertEquals(1, respone.getOmVolumeArgs().getUpdateID());
} catch (IllegalArgumentException ex){ } catch (IllegalArgumentException ex){
GenericTestUtils.assertExceptionContains("should be greater than zero", GenericTestUtils.assertExceptionContains("should be greater than zero",
ex); ex);
@ -106,7 +112,7 @@ public void testValidateAndUpdateCacheSuccess() throws Exception {
omVolumeCreateRequest = new OMVolumeCreateRequest(modifiedRequest); omVolumeCreateRequest = new OMVolumeCreateRequest(modifiedRequest);
OMClientResponse omClientResponse = OMClientResponse omClientResponse =
omVolumeCreateRequest.validateAndUpdateCache(ozoneManager, 1, omVolumeCreateRequest.validateAndUpdateCache(ozoneManager, 2,
ozoneManagerDoubleBufferHelper); ozoneManagerDoubleBufferHelper);
OzoneManagerProtocolProtos.OMResponse omResponse = OzoneManagerProtocolProtos.OMResponse omResponse =
@ -124,6 +130,8 @@ public void testValidateAndUpdateCacheSuccess() throws Exception {
omMetadataManager.getVolumeTable().get(volumeKey); omMetadataManager.getVolumeTable().get(volumeKey);
// As request is valid volume table should not have entry. // As request is valid volume table should not have entry.
Assert.assertNotNull(omVolumeArgs); Assert.assertNotNull(omVolumeArgs);
Assert.assertEquals(2, omVolumeArgs.getObjectID());
Assert.assertEquals(2, omVolumeArgs.getUpdateID());
// Check data from table and request. // Check data from table and request.
Assert.assertEquals(volumeInfo.getVolume(), omVolumeArgs.getVolume()); Assert.assertEquals(volumeInfo.getVolume(), omVolumeArgs.getVolume());