From 06998a11266c8d71a67114ef5c9a691987426630 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Thu, 26 Sep 2019 09:41:03 -0700 Subject: [PATCH] HDDS-2180. Add Object ID and update ID on VolumeList Object. (#1526) --- .../src/main/proto/OzoneManagerProtocol.proto | 2 ++ .../s3/bucket/S3BucketCreateRequest.java | 3 ++- .../request/volume/OMVolumeCreateRequest.java | 2 +- .../request/volume/OMVolumeDeleteRequest.java | 3 ++- .../om/request/volume/OMVolumeRequest.java | 18 ++++++++++++++---- .../volume/OMVolumeSetOwnerRequest.java | 6 ++++-- .../ozone/om/request/TestOMRequestUtils.java | 8 ++++++-- .../ozone/om/response/TestOMResponseUtils.java | 2 ++ .../volume/TestOMVolumeCreateResponse.java | 1 + .../volume/TestOMVolumeDeleteResponse.java | 5 ++++- .../volume/TestOMVolumeSetOwnerResponse.java | 9 ++++++++- 11 files changed, 46 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto index 87472833b2..2b119a68da 100644 --- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto @@ -354,6 +354,8 @@ message CreateVolumeResponse { message VolumeList { repeated string volumeNames = 1; + optional uint64 objectID = 2; + optional uint64 updateID = 3; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java index f28f11b7fd..73772982a7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java @@ -179,7 +179,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, VolumeList volumeList = omMetadataManager.getUserTable().get( omMetadataManager.getUserKey(userName)); volumeList = addVolumeToOwnerList(volumeList, - volumeName, userName, ozoneManager.getMaxUserVolumeCount()); + volumeName, userName, ozoneManager.getMaxUserVolumeCount(), + transactionLogIndex); createVolume(omMetadataManager, omVolumeArgs, volumeList, volumeKey, omMetadataManager.getUserKey(userName), transactionLogIndex); volumeCreated = true; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java index 9b8a6fa5b9..3568b3ba23 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java @@ -151,7 +151,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, String dbUserKey = omMetadataManager.getUserKey(owner); volumeList = omMetadataManager.getUserTable().get(dbUserKey); volumeList = addVolumeToOwnerList(volumeList, volume, owner, - ozoneManager.getMaxUserVolumeCount()); + ozoneManager.getMaxUserVolumeCount(), transactionLogIndex); createVolume(omMetadataManager, omVolumeArgs, volumeList, dbVolumeKey, dbUserKey, transactionLogIndex); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeDeleteRequest.java index 9a1c422c89..9bd9a1d5ae 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeDeleteRequest.java @@ -115,7 +115,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // delete the volume from the owner list // as well as delete the volume entry - newVolumeList = delVolumeFromOwnerList(newVolumeList, volume, owner); + newVolumeList = delVolumeFromOwnerList(newVolumeList, volume, owner, + transactionLogIndex); omMetadataManager.getUserTable().addCacheEntry(new CacheKey<>(dbUserKey), new CacheValue<>(Optional.of(newVolumeList), transactionLogIndex)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeRequest.java index dc44375fe3..a5e7645b45 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeRequest.java @@ -48,12 +48,13 @@ public OMVolumeRequest(OMRequest omRequest) { * acquiring user lock. * @param volumeList - current volume list owned by user. * @param volume - volume which needs to deleted from the volume list. - * @param owner + * @param owner - Name of the Owner. + * @param txID - The transaction ID that is updating this value. * @return VolumeList - updated volume list for the user. * @throws IOException */ protected VolumeList delVolumeFromOwnerList(VolumeList volumeList, - String volume, String owner) throws IOException { + String volume, String owner, long txID) throws IOException { List prevVolList = new ArrayList<>(); @@ -68,7 +69,10 @@ protected VolumeList delVolumeFromOwnerList(VolumeList volumeList, // Remove the volume from the list prevVolList.remove(volume); VolumeList newVolList = VolumeList.newBuilder() - .addAllVolumeNames(prevVolList).build(); + .addAllVolumeNames(prevVolList) + .setObjectID(volumeList.getObjectID()) + .setUpdateID(txID) + .build(); return newVolList; } @@ -85,7 +89,8 @@ protected VolumeList delVolumeFromOwnerList(VolumeList volumeList, * maxUserVolumeCount, an exception is thrown. */ protected VolumeList addVolumeToOwnerList(VolumeList volumeList, - String volume, String owner, long maxUserVolumeCount) throws IOException { + String volume, String owner, long maxUserVolumeCount, long txID) + throws IOException { // Check the volume count if (volumeList != null && @@ -95,13 +100,18 @@ protected VolumeList addVolumeToOwnerList(VolumeList volumeList, } List prevVolList = new ArrayList<>(); + long objectID = txID; if (volumeList != null) { prevVolList.addAll(volumeList.getVolumeNamesList()); + objectID = volumeList.getObjectID(); } + // Add the new volume to the list prevVolList.add(volume); VolumeList newVolList = VolumeList.newBuilder() + .setObjectID(objectID) + .setUpdateID(txID) .addAllVolumeNames(prevVolList).build(); return newVolList; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java index 3fdb152792..0dd0ff7bf5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java @@ -144,14 +144,16 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omMetadataManager.getUserTable().get(oldOwner); oldOwnerVolumeList = delVolumeFromOwnerList( - oldOwnerVolumeList, volume, oldOwner); + oldOwnerVolumeList, volume, oldOwner, transactionLogIndex); newOwnerVolumeList = omMetadataManager.getUserTable().get(newOwner); newOwnerVolumeList = addVolumeToOwnerList( - newOwnerVolumeList, volume, newOwner, maxUserVolumeCount); + newOwnerVolumeList, volume, newOwner, + maxUserVolumeCount, transactionLogIndex); // Set owner with new owner name. omVolumeArgs.setOwnerName(newOwner); + omVolumeArgs.setUpdateID(transactionLogIndex); // Update cache. omMetadataManager.getUserTable().addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java index 72e049bb42..20ff425b71 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java @@ -266,8 +266,12 @@ public static List< HddsProtos.KeyValue> getMetadataList() { public static void addUserToDB(String volumeName, String ownerName, OMMetadataManager omMetadataManager) throws Exception { OzoneManagerProtocolProtos.VolumeList volumeList = - OzoneManagerProtocolProtos.VolumeList.newBuilder() - .addVolumeNames(volumeName).build(); + OzoneManagerProtocolProtos.VolumeList + .newBuilder() + .addVolumeNames(volumeName) + .setObjectID(1) + .setUpdateID(1) + .build(); omMetadataManager.getUserTable().put( omMetadataManager.getUserKey(ownerName), volumeList); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestOMResponseUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestOMResponseUtils.java index 4cafe7a670..2f4dbb94a9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestOMResponseUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestOMResponseUtils.java @@ -58,6 +58,8 @@ public static S3BucketCreateResponse createS3BucketResponse(String userName, OzoneManagerProtocolProtos.VolumeList volumeList = OzoneManagerProtocolProtos.VolumeList.newBuilder() + .setObjectID(1) + .setUpdateID(1) .addVolumeNames(volumeName).build(); OmVolumeArgs omVolumeArgs = OmVolumeArgs.newBuilder() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java index f395e0128c..f6d18883bc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java @@ -69,6 +69,7 @@ public void testAddToDBBatch() throws Exception { String volumeName = UUID.randomUUID().toString(); String userName = "user1"; VolumeList volumeList = VolumeList.newBuilder() + .setObjectID(1).setUpdateID(1) .addVolumeNames(volumeName).build(); OMResponse omResponse = OMResponse.newBuilder() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java index 67bdf15308..12e7388f83 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java @@ -69,6 +69,8 @@ public void testAddToDBBatch() throws Exception { String volumeName = UUID.randomUUID().toString(); String userName = "user1"; VolumeList volumeList = VolumeList.newBuilder() + .setObjectID(1) + .setUpdateID(1) .addVolumeNames(volumeName).build(); OMResponse omResponse = OMResponse.newBuilder() @@ -85,7 +87,8 @@ public void testAddToDBBatch() throws Exception { new OMVolumeCreateResponse(omVolumeArgs, volumeList, omResponse); // As we are deleting updated volume list should be empty. - VolumeList updatedVolumeList = VolumeList.newBuilder().build(); + VolumeList updatedVolumeList = VolumeList.newBuilder() + .setObjectID(1).setUpdateID(1).build(); OMVolumeDeleteResponse omVolumeDeleteResponse = new OMVolumeDeleteResponse(volumeName, userName, updatedVolumeList, omResponse); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java index a04df9ccd4..38255bac35 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java @@ -70,6 +70,8 @@ public void testAddToDBBatch() throws Exception { String volumeName = UUID.randomUUID().toString(); String oldOwner = "user1"; VolumeList volumeList = VolumeList.newBuilder() + .setObjectID(1) + .setUpdateID(1) .addVolumeNames(volumeName).build(); OMResponse omResponse = OMResponse.newBuilder() @@ -89,8 +91,13 @@ public void testAddToDBBatch() throws Exception { String newOwner = "user2"; VolumeList newOwnerVolumeList = VolumeList.newBuilder() + .setObjectID(1) + .setUpdateID(1) .addVolumeNames(volumeName).build(); - VolumeList oldOwnerVolumeList = VolumeList.newBuilder().build(); + VolumeList oldOwnerVolumeList = VolumeList.newBuilder() + .setObjectID(2) + .setUpdateID(2) + .build(); OmVolumeArgs newOwnerVolumeArgs = OmVolumeArgs.newBuilder() .setOwnerName(newOwner).setAdminName(newOwner) .setVolume(volumeName).setCreationTime(omVolumeArgs.getCreationTime())