big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-2161. Create RepeatedKeyInfo structure to be saved in deletedTable

Browse Source 
Signed-off-by: Anu Engineer <aengineer@apache.org>
This commit is contained in:
dchitlangia 2019-09-20 18:06:30 -04:00 committed by Anu Engineer
parent 3d78b1223d
commit 3fd3d746fc
16 changed files with 324 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
View File

@ -26,9 +26,11 @@
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.lock.OzoneManagerLock;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.VolumeList;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.Table;
@ -251,7 +253,7 @@ List<OmVolumeArgs> listVolumes(String userName, String prefix,
*
* @return Deleted Table.
*/
Table<String, OmKeyInfo> getDeletedTable();
Table<String, RepeatedOmKeyInfo> getDeletedTable();
/**
* Gets the OpenKeyTable.

52
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/RepeatedOmKeyInfoCodec.java Normal file
View File

@ -0,0 +1,52 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.om.codec;
import com.google.common.base.Preconditions;
import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.RepeatedKeyInfo;
import java.io.IOException;
/**
* Codec to encode RepeatedOmKeyInfo as byte array.
*/
public class RepeatedOmKeyInfoCodec implements Codec<RepeatedOmKeyInfo> {
@Override
public byte[] toPersistedFormat(RepeatedOmKeyInfo object)
throws IOException {
Preconditions.checkNotNull(object,
"Null object can't be converted to byte array.");
return object.getProto().toByteArray();
}
@Override
public RepeatedOmKeyInfo fromPersistedFormat(byte[] rawData)
throws IOException {
Preconditions.checkNotNull(rawData,
"Null byte array can't converted to real object.");
try {
return RepeatedOmKeyInfo.getFromProto(RepeatedKeyInfo.parseFrom(rawData));
} catch (InvalidProtocolBufferException e) {
throw new IllegalArgumentException(
"Can't encode the the raw data from the byte array", e);
}
}
}

91
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java Normal file
View File

@ -0,0 +1,91 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.om.helpers;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.RepeatedKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.KeyInfo;
/**
* Args for deleted keys. This is written to om metadata deletedTable.
* Once a key is deleted, it is moved to om metadata deletedTable. Having a
* {label: List<OMKeyInfo>} ensures that if users create & delete keys with
* exact same uri multiple times, all the delete instances are bundled under
* the same key name. This is useful as part of GDPR compliance where an
* admin wants to confirm if a given key is deleted from deletedTable metadata.
*/
public class RepeatedOmKeyInfo {
private List<OmKeyInfo> omKeyInfoList;
public RepeatedOmKeyInfo(List<OmKeyInfo> omKeyInfos) {
this.omKeyInfoList = omKeyInfos;
}
public RepeatedOmKeyInfo(OmKeyInfo omKeyInfos) {
this.omKeyInfoList = new ArrayList<>();
this.omKeyInfoList.add(omKeyInfos);
}
public void addOmKeyInfo(OmKeyInfo info) {
this.omKeyInfoList.add(info);
}
public List<OmKeyInfo> getOmKeyInfoList() {
return omKeyInfoList;
}
public static RepeatedOmKeyInfo getFromProto(RepeatedKeyInfo
repeatedKeyInfo) {
List<OmKeyInfo> list = new ArrayList<>();
for(KeyInfo k : repeatedKeyInfo.getKeyInfoList()) {
list.add(OmKeyInfo.getFromProtobuf(k));
}
return new RepeatedOmKeyInfo.Builder().setOmKeyInfos(list).build();
}
public RepeatedKeyInfo getProto() {
List<KeyInfo> list = new ArrayList<>();
for(OmKeyInfo k : omKeyInfoList) {
list.add(k.getProtobuf());
}
RepeatedKeyInfo.Builder builder = RepeatedKeyInfo.newBuilder()
.addAllKeyInfo(list);
return builder.build();
}
/**
* Builder of RepeatedOmKeyInfo.
*/
public static class Builder {
private List<OmKeyInfo> omKeyInfos;
public Builder(){}
public Builder setOmKeyInfos(List<OmKeyInfo> infoList) {
this.omKeyInfos = infoList;
return this;
}
public RepeatedOmKeyInfo build() {
return new RepeatedOmKeyInfo(omKeyInfos);
}
}
}

4
hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
View File

@ -687,6 +687,10 @@ message KeyInfo {
repeated OzoneAclInfo acls = 13;
}
message RepeatedKeyInfo {
repeated KeyInfo keyInfo = 1;
}
message OzoneFileStatusProto {
required hadoop.fs.FileStatusProto status = 1;
}

47
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
View File

@ -85,6 +85,7 @@
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
import org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
@ -781,9 +782,17 @@ public void deleteKey(OmKeyArgs args) throws IOException {
return;
}
}
metadataManager.getStore().move(objectKey,
metadataManager.getKeyTable(),
metadataManager.getDeletedTable());
//Check if key with same keyName exists in deletedTable and then
// insert/update accordingly.
RepeatedOmKeyInfo repeatedOmKeyInfo =
metadataManager.getDeletedTable().get(objectKey);
if(repeatedOmKeyInfo == null) {
repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyInfo);
} else {
repeatedOmKeyInfo.addOmKeyInfo(keyInfo);
}
metadataManager.getKeyTable().delete(objectKey);
metadataManager.getDeletedTable().put(objectKey, repeatedOmKeyInfo);
} catch (OMException ex) {
throw ex;
} catch (IOException ex) {
@ -1003,7 +1012,14 @@ public OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
// will not be garbage collected, so move this part to delete table
// and throw error
// Move this part to delete table.
metadataManager.getDeletedTable().put(partName, keyInfo);
RepeatedOmKeyInfo repeatedOmKeyInfo =
metadataManager.getDeletedTable().get(partName);
if(repeatedOmKeyInfo == null) {
repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyInfo);
} else {
repeatedOmKeyInfo.addOmKeyInfo(keyInfo);
}
metadataManager.getDeletedTable().put(partName, repeatedOmKeyInfo);
throw new OMException("No such Multipart upload is with specified " +
"uploadId " + uploadID, ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR);
} else {
@ -1031,9 +1047,19 @@ public OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
// Add the new entry in to the list of part keys.
DBStore store = metadataManager.getStore();
try (BatchOperation batch = store.initBatchOperation()) {
RepeatedOmKeyInfo repeatedOmKeyInfo = metadataManager.
getDeletedTable().get(oldPartKeyInfo.getPartName());
if(repeatedOmKeyInfo == null) {
repeatedOmKeyInfo = new RepeatedOmKeyInfo(
OmKeyInfo.getFromProtobuf(oldPartKeyInfo.getPartKeyInfo()));
} else {
repeatedOmKeyInfo.addOmKeyInfo(
OmKeyInfo.getFromProtobuf(oldPartKeyInfo.getPartKeyInfo()));
}
metadataManager.getDeletedTable().put(partName, repeatedOmKeyInfo);
metadataManager.getDeletedTable().putWithBatch(batch,
oldPartKeyInfo.getPartName(),
OmKeyInfo.getFromProtobuf(oldPartKeyInfo.getPartKeyInfo()));
repeatedOmKeyInfo);
metadataManager.getOpenKeyTable().deleteWithBatch(batch, openKey);
metadataManager.getMultipartInfoTable().putWithBatch(batch,
multipartKey, multipartKeyInfo);
@ -1252,8 +1278,17 @@ public void abortMultipartUpload(OmKeyArgs omKeyArgs) throws IOException {
PartKeyInfo partKeyInfo = partKeyInfoEntry.getValue();
OmKeyInfo currentKeyPartInfo = OmKeyInfo.getFromProtobuf(
partKeyInfo.getPartKeyInfo());
RepeatedOmKeyInfo repeatedOmKeyInfo = metadataManager.
getDeletedTable().get(partKeyInfo.getPartName());
if(repeatedOmKeyInfo == null) {
repeatedOmKeyInfo = new RepeatedOmKeyInfo(currentKeyPartInfo);
} else {
repeatedOmKeyInfo.addOmKeyInfo(currentKeyPartInfo);
}
metadataManager.getDeletedTable().putWithBatch(batch,
partKeyInfo.getPartName(), currentKeyPartInfo);
partKeyInfo.getPartName(), repeatedOmKeyInfo);
}
// Finally delete the entry from the multipart info table and open
// key table

90
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
View File

@ -44,6 +44,7 @@
import org.apache.hadoop.ozone.om.codec.OmMultipartKeyInfoCodec;
import org.apache.hadoop.ozone.om.codec.OmPrefixInfoCodec;
import org.apache.hadoop.ozone.om.codec.OmVolumeArgsCodec;
import org.apache.hadoop.ozone.om.codec.RepeatedOmKeyInfoCodec;
import org.apache.hadoop.ozone.om.codec.S3SecretValueCodec;
import org.apache.hadoop.ozone.om.codec.TokenIdentifierCodec;
import org.apache.hadoop.ozone.om.codec.VolumeListCodec;
@ -57,6 +58,7 @@
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.lock.OzoneManagerLock;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList;
@ -87,31 +89,31 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
* OM DB stores metadata as KV pairs in different column families.
* <p>
* OM DB Schema:
* |-------------------------------------------------------------------|
* | Column Family | VALUE |
* |-------------------------------------------------------------------|
* | userTable | user->VolumeList |
* |-------------------------------------------------------------------|
* | volumeTable | /volume->VolumeInfo |
* |-------------------------------------------------------------------|
* | bucketTable | /volume/bucket-> BucketInfo |
* |-------------------------------------------------------------------|
* | keyTable | /volumeName/bucketName/keyName->KeyInfo |
* |-------------------------------------------------------------------|
* | deletedTable | /volumeName/bucketName/keyName->KeyInfo |
* |-------------------------------------------------------------------|
* | openKey | /volumeName/bucketName/keyName/id->KeyInfo |
* |-------------------------------------------------------------------|
* | s3Table | s3BucketName -> /volumeName/bucketName |
* |-------------------------------------------------------------------|
* | s3SecretTable | s3g_access_key_id -> s3Secret |
* |-------------------------------------------------------------------|
* | dTokenTable | s3g_access_key_id -> s3Secret |
* |-------------------------------------------------------------------|
* | prefixInfoTable | prefix -> PrefixInfo |
* |-------------------------------------------------------------------|
* | multipartInfoTable| /volumeName/bucketName/keyName/uploadId ->...|
* |-------------------------------------------------------------------|
* |----------------------------------------------------------------------|
* | Column Family | VALUE |
* |----------------------------------------------------------------------|
* | userTable | user->VolumeList |
* |----------------------------------------------------------------------|
* | volumeTable | /volume->VolumeInfo |
* |----------------------------------------------------------------------|
* | bucketTable | /volume/bucket-> BucketInfo |
* |----------------------------------------------------------------------|
* | keyTable | /volumeName/bucketName/keyName->KeyInfo |
* |----------------------------------------------------------------------|
* | deletedTable | /volumeName/bucketName/keyName->RepeatedKeyInfo |
* |----------------------------------------------------------------------|
* | openKey | /volumeName/bucketName/keyName/id->KeyInfo |
* |----------------------------------------------------------------------|
* | s3Table | s3BucketName -> /volumeName/bucketName |
* |----------------------------------------------------------------------|
* | s3SecretTable | s3g_access_key_id -> s3Secret |
* |----------------------------------------------------------------------|
* | dTokenTable | s3g_access_key_id -> s3Secret |
* |----------------------------------------------------------------------|
* | prefixInfoTable | prefix -> PrefixInfo |
* |----------------------------------------------------------------------|
* | multipartInfoTable| /volumeName/bucketName/keyName/uploadId ->... |
* |----------------------------------------------------------------------|
*/
public static final String USER_TABLE = "userTable";
@ -192,7 +194,7 @@ public Table<String, OmKeyInfo> getKeyTable() {
}
@Override
public Table<String, OmKeyInfo> getDeletedTable() {
public Table<String, RepeatedOmKeyInfo> getDeletedTable() {
return deletedTable;
}
@ -261,6 +263,7 @@ protected DBStoreBuilder addOMTablesAndCodecs(DBStoreBuilder builder) {
.addTable(PREFIX_TABLE)
.addCodec(OzoneTokenIdentifier.class, new TokenIdentifierCodec())
.addCodec(OmKeyInfo.class, new OmKeyInfoCodec())
.addCodec(RepeatedOmKeyInfo.class, new RepeatedOmKeyInfoCodec())
.addCodec(OmBucketInfo.class, new OmBucketInfoCodec())
.addCodec(OmVolumeArgs.class, new OmVolumeArgsCodec())
.addCodec(VolumeList.class, new VolumeListCodec())
@ -296,8 +299,8 @@ protected void initializeOmTables() throws IOException {
keyTable = this.store.getTable(KEY_TABLE, String.class, OmKeyInfo.class);
checkTableStatus(keyTable, KEY_TABLE);
deletedTable =
this.store.getTable(DELETED_TABLE, String.class, OmKeyInfo.class);
deletedTable = this.store.getTable(DELETED_TABLE, String.class,
RepeatedOmKeyInfo.class);
checkTableStatus(deletedTable, DELETED_TABLE);
openKeyTable =
@ -765,25 +768,26 @@ private VolumeList getVolumesByUser(String userNameKey)
public List<BlockGroup> getPendingDeletionKeys(final int keyCount)
throws IOException {
List<BlockGroup> keyBlocksList = Lists.newArrayList();
try (TableIterator<String, ? extends KeyValue<String, OmKeyInfo>> keyIter =
getDeletedTable()
.iterator()) {
try (TableIterator<String, ? extends KeyValue<String, RepeatedOmKeyInfo>>
keyIter = getDeletedTable().iterator()) {
int currentCount = 0;
while (keyIter.hasNext() && currentCount < keyCount) {
KeyValue<String, OmKeyInfo> kv = keyIter.next();
KeyValue<String, RepeatedOmKeyInfo> kv = keyIter.next();
if (kv != null) {
OmKeyInfo info = kv.getValue();
RepeatedOmKeyInfo infoList = kv.getValue();
// Get block keys as a list.
OmKeyLocationInfoGroup latest = info.getLatestVersionLocations();
List<BlockID> item = latest.getLocationList().stream()
.map(b -> new BlockID(b.getContainerID(), b.getLocalID()))
.collect(Collectors.toList());
BlockGroup keyBlocks = BlockGroup.newBuilder()
.setKeyName(kv.getKey())
.addAllBlockIDs(item)
.build();
keyBlocksList.add(keyBlocks);
currentCount++;
for(OmKeyInfo info : infoList.getOmKeyInfoList()){
OmKeyLocationInfoGroup latest = info.getLatestVersionLocations();
List<BlockID> item = latest.getLocationList().stream()
.map(b -> new BlockID(b.getContainerID(), b.getLocalID()))
.collect(Collectors.toList());
BlockGroup keyBlocks = BlockGroup.newBuilder()
.setKeyName(kv.getKey())
.addAllBlockIDs(item)
.build();
keyBlocksList.add(keyBlocks);
currentCount++;
}
}
}
}

3
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
View File

@ -139,13 +139,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
// TODO: Revisit if we need it later.
omClientResponse = new OMKeyDeleteResponse(omKeyInfo,
deleteKeyArgs.getModificationTime(),
omResponse.setDeleteKeyResponse(
DeleteKeyResponse.newBuilder()).build());
} catch (IOException ex) {
exception = ex;
omClientResponse = new OMKeyDeleteResponse(null, 0,
omClientResponse = new OMKeyDeleteResponse(null,
createErrorOMResponse(omResponse, exception));
} finally {
if (omClientResponse != null) {

5
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
View File

@ -133,14 +133,13 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
}
omClientResponse = new S3MultipartUploadAbortResponse(multipartKey,
keyArgs.getModificationTime(), multipartKeyInfo,
multipartKeyInfo,
omResponse.setAbortMultiPartUploadResponse(
MultipartUploadAbortResponse.newBuilder()).build());
} catch (IOException ex) {
exception = ex;
omClientResponse = new S3MultipartUploadAbortResponse(multipartKey,
keyArgs.getModificationTime(), multipartKeyInfo,
createErrorOMResponse(omResponse, exception));
multipartKeyInfo, createErrorOMResponse(omResponse, exception));
} finally {
if (omClientResponse != null) {
omClientResponse.setFlushFuture(

4
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java
View File

@ -188,13 +188,13 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
omResponse.setCommitMultiPartUploadResponse(
MultipartCommitUploadPartResponse.newBuilder().setPartName(partName));
omClientResponse = new S3MultipartUploadCommitPartResponse(multipartKey,
openKey, keyArgs.getModificationTime(), omKeyInfo, multipartKeyInfo,
openKey, omKeyInfo, multipartKeyInfo,
oldPartKeyInfo, omResponse.build());
} catch (IOException ex) {
exception = ex;
omClientResponse = new S3MultipartUploadCommitPartResponse(multipartKey,
openKey, keyArgs.getModificationTime(), omKeyInfo, multipartKeyInfo,
openKey, omKeyInfo, multipartKeyInfo,
oldPartKeyInfo, createErrorOMResponse(omResponse, exception));
} finally {
if (omClientResponse != null) {

27
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java
View File

@ -18,10 +18,10 @@
package org.apache.hadoop.ozone.om.response.key;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@ -35,13 +35,10 @@
*/
public class OMKeyDeleteResponse extends OMClientResponse {
private OmKeyInfo omKeyInfo;
private long deleteTimestamp;
public OMKeyDeleteResponse(OmKeyInfo omKeyInfo, long deletionTime,
OMResponse omResponse) {
public OMKeyDeleteResponse(OmKeyInfo omKeyInfo, OMResponse omResponse) {
super(omResponse);
this.omKeyInfo = omKeyInfo;
this.deleteTimestamp = deletionTime;
}
@Override
@ -60,12 +57,22 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
if (!isKeyEmpty(omKeyInfo)) {
// If a deleted key is put in the table where a key with the same
// name already exists, then the old deleted key information would be
// lost. To differentiate between keys with same name in
// deletedTable, we add the timestamp to the key name.
String deleteKeyName = OmUtils.getDeletedKeyName(
ozoneKey, deleteTimestamp);
// lost. To avoid this, first check if a key with same name exists.
// deletedTable in OM Metadata stores <KeyName, RepeatedOMKeyInfo>.
// The RepeatedOmKeyInfo is the structure that allows us to store a
// list of OmKeyInfo that can be tied to same key name. For a keyName
// if RepeatedOMKeyInfo structure is null, we create a new instance,
// if it is not null, then we simply add to the list and store this
// instance in deletedTable.
RepeatedOmKeyInfo repeatedOmKeyInfo =
omMetadataManager.getDeletedTable().get(ozoneKey);
if(repeatedOmKeyInfo == null) {
repeatedOmKeyInfo = new RepeatedOmKeyInfo(omKeyInfo);
} else {
repeatedOmKeyInfo.addOmKeyInfo(omKeyInfo);
}
omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
deleteKeyName, omKeyInfo);
ozoneKey, repeatedOmKeyInfo);
}
}
}

21
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadAbortResponse.java
View File

@ -18,10 +18,10 @@
package org.apache.hadoop.ozone.om.response.s3.multipart;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@ -40,16 +40,12 @@
public class S3MultipartUploadAbortResponse extends OMClientResponse {
private String multipartKey;
private long timeStamp;
private OmMultipartKeyInfo omMultipartKeyInfo;
public S3MultipartUploadAbortResponse(String multipartKey,
long timeStamp,
OmMultipartKeyInfo omMultipartKeyInfo,
OMResponse omResponse) {
OmMultipartKeyInfo omMultipartKeyInfo, OMResponse omResponse) {
super(omResponse);
this.multipartKey = multipartKey;
this.timeStamp = timeStamp;
this.omMultipartKeyInfo = omMultipartKeyInfo;
}
@ -73,9 +69,18 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
PartKeyInfo partKeyInfo = partKeyInfoEntry.getValue();
OmKeyInfo currentKeyPartInfo =
OmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo());
RepeatedOmKeyInfo repeatedOmKeyInfo =
omMetadataManager.getDeletedTable().get(partKeyInfo.getPartName());
if(repeatedOmKeyInfo == null) {
repeatedOmKeyInfo = new RepeatedOmKeyInfo(currentKeyPartInfo);
} else {
repeatedOmKeyInfo.addOmKeyInfo(currentKeyPartInfo);
}
omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
OmUtils.getDeletedKeyName(partKeyInfo.getPartName(), timeStamp),
currentKeyPartInfo);
partKeyInfo.getPartName(),
repeatedOmKeyInfo);
}
}

34
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java
View File

@ -18,10 +18,10 @@
package org.apache.hadoop.ozone.om.response.s3.multipart;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@ -41,21 +41,19 @@ public class S3MultipartUploadCommitPartResponse extends OMClientResponse {
private String multipartKey;
private String openKey;
private long deleteTimeStamp;
private OmKeyInfo deletePartKeyInfo;
private OmMultipartKeyInfo omMultipartKeyInfo;
private OzoneManagerProtocolProtos.PartKeyInfo oldMultipartKeyInfo;
public S3MultipartUploadCommitPartResponse(String multipartKey,
String openKey, long deleteTimeStamp,
OmKeyInfo deletePartKeyInfo, OmMultipartKeyInfo omMultipartKeyInfo,
String openKey, OmKeyInfo deletePartKeyInfo,
OmMultipartKeyInfo omMultipartKeyInfo,
OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo,
OMResponse omResponse) {
super(omResponse);
this.multipartKey = multipartKey;
this.openKey = openKey;
this.deleteTimeStamp = deleteTimeStamp;
this.deletePartKeyInfo = deletePartKeyInfo;
this.omMultipartKeyInfo = omMultipartKeyInfo;
this.oldMultipartKeyInfo = oldPartKeyInfo;
@ -69,9 +67,16 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
if (getOMResponse().getStatus() == NO_SUCH_MULTIPART_UPLOAD_ERROR) {
// Means by the time we try to commit part, some one has aborted this
// multipart upload. So, delete this part information.
RepeatedOmKeyInfo repeatedOmKeyInfo =
omMetadataManager.getDeletedTable().get(openKey);
if(repeatedOmKeyInfo == null) {
repeatedOmKeyInfo = new RepeatedOmKeyInfo(deletePartKeyInfo);
} else {
repeatedOmKeyInfo.addOmKeyInfo(deletePartKeyInfo);
}
omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
OmUtils.getDeletedKeyName(openKey, deleteTimeStamp),
deletePartKeyInfo);
openKey,
repeatedOmKeyInfo);
}
if (getOMResponse().getStatus() == OK) {
@ -85,10 +90,19 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
// This means for this multipart upload part upload, we have an old
// part information, so delete it.
if (oldMultipartKeyInfo != null) {
RepeatedOmKeyInfo repeatedOmKeyInfo = omMetadataManager.
getDeletedTable().get(oldMultipartKeyInfo.getPartName());
if(repeatedOmKeyInfo == null) {
repeatedOmKeyInfo = new RepeatedOmKeyInfo(
OmKeyInfo.getFromProtobuf(oldMultipartKeyInfo.getPartKeyInfo()));
} else {
repeatedOmKeyInfo.addOmKeyInfo(
OmKeyInfo.getFromProtobuf(oldMultipartKeyInfo.getPartKeyInfo()));
}
omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
OmUtils.getDeletedKeyName(oldMultipartKeyInfo.getPartName(),
deleteTimeStamp),
OmKeyInfo.getFromProtobuf(oldMultipartKeyInfo.getPartKeyInfo()));
oldMultipartKeyInfo.getPartName(),
repeatedOmKeyInfo);
}

14
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
View File

@ -28,13 +28,13 @@
import com.google.common.base.Optional;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.request.s3.bucket.S3BucketCreateRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@ -370,10 +370,16 @@ public static String deleteKey(String ozoneKey,
// Delete key from KeyTable and put in DeletedKeyTable
omMetadataManager.getKeyTable().delete(ozoneKey);
String deletedKeyName = OmUtils.getDeletedKeyName(ozoneKey, Time.now());
omMetadataManager.getDeletedTable().put(deletedKeyName, omKeyInfo);
RepeatedOmKeyInfo repeatedOmKeyInfo =
omMetadataManager.getDeletedTable().get(ozoneKey);
if(repeatedOmKeyInfo == null) {
repeatedOmKeyInfo = new RepeatedOmKeyInfo(omKeyInfo);
} else {
repeatedOmKeyInfo.addOmKeyInfo(omKeyInfo);
}
omMetadataManager.getDeletedTable().put(ozoneKey, repeatedOmKeyInfo);
return deletedKeyName;
return ozoneKey;
}
/**

20
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponse.java
View File

@ -21,9 +21,7 @@
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.util.Time;
import org.junit.Assert;
import org.junit.Test;
@ -52,15 +50,11 @@ public void testAddToDBBatch() throws Exception {
.setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey)
.build();
long deletionTime = Time.now();
OMKeyDeleteResponse omKeyDeleteResponse =
new OMKeyDeleteResponse(omKeyInfo, deletionTime, omResponse);
new OMKeyDeleteResponse(omKeyInfo, omResponse);
String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
keyName);
String deletedOzoneKeyName = OmUtils.getDeletedKeyName(
ozoneKey, deletionTime);
TestOMRequestUtils.addKeyToTable(false, volumeName, bucketName, keyName,
clientID, replicationType, replicationFactor, omMetadataManager);
@ -76,7 +70,7 @@ public void testAddToDBBatch() throws Exception {
// As default key entry does not have any blocks, it should not be in
// deletedKeyTable.
Assert.assertFalse(omMetadataManager.getDeletedTable().isExist(
deletedOzoneKeyName));
ozoneKey));
}
@Test
@ -117,13 +111,9 @@ public void testAddToDBBatchWithNonEmptyBlocks() throws Exception {
.setStatus(OzoneManagerProtocolProtos.Status.OK)
.setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey)
.build();
long deletionTime = Time.now();
OMKeyDeleteResponse omKeyDeleteResponse =
new OMKeyDeleteResponse(omKeyInfo, deletionTime, omResponse);
String deletedOzoneKeyName = OmUtils.getDeletedKeyName(
ozoneKey, deletionTime);
new OMKeyDeleteResponse(omKeyInfo, omResponse);
Assert.assertTrue(omMetadataManager.getKeyTable().isExist(ozoneKey));
omKeyDeleteResponse.addToDBBatch(omMetadataManager, batchOperation);
@ -135,7 +125,7 @@ public void testAddToDBBatchWithNonEmptyBlocks() throws Exception {
// Key has blocks, it should not be in deletedKeyTable.
Assert.assertTrue(omMetadataManager.getDeletedTable().isExist(
deletedOzoneKeyName));
ozoneKey));
}
@ -152,7 +142,7 @@ public void testAddToDBBatchWithErrorResponse() throws Exception {
.build();
OMKeyDeleteResponse omKeyDeleteResponse =
new OMKeyDeleteResponse(omKeyInfo, Time.now(), omResponse);
new OMKeyDeleteResponse(omKeyInfo, omResponse);
String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
keyName);

3
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java
View File

@ -113,8 +113,7 @@ public S3MultipartUploadAbortResponse createS3AbortMPUResponse(
.setAbortMultiPartUploadResponse(
MultipartUploadAbortResponse.newBuilder().build()).build();
return new S3MultipartUploadAbortResponse(multipartKey, timeStamp,
omMultipartKeyInfo,
return new S3MultipartUploadAbortResponse(multipartKey, omMultipartKeyInfo,
omResponse);
}

19
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponse.java
View File

@ -20,10 +20,10 @@
import java.util.UUID;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.junit.Assert;
import org.junit.Test;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@ -124,24 +124,25 @@ public void testAddDBToBatchWithParts() throws Exception {
Assert.assertTrue(omMetadataManager.countRowsInTable(
omMetadataManager.getDeletedTable()) == 2);
String part1DeletedKeyName = OmUtils.getDeletedKeyName(
omMultipartKeyInfo.getPartKeyInfo(1).getPartName(),
timeStamp);
String part1DeletedKeyName =
omMultipartKeyInfo.getPartKeyInfo(1).getPartName();
String part2DeletedKeyName = OmUtils.getDeletedKeyName(
omMultipartKeyInfo.getPartKeyInfo(2).getPartName(),
timeStamp);
String part2DeletedKeyName =
omMultipartKeyInfo.getPartKeyInfo(2).getPartName();
Assert.assertNotNull(omMetadataManager.getDeletedTable().get(
part1DeletedKeyName));
Assert.assertNotNull(omMetadataManager.getDeletedTable().get(
part2DeletedKeyName));
RepeatedOmKeyInfo ro =
omMetadataManager.getDeletedTable().get(part1DeletedKeyName);
Assert.assertEquals(OmKeyInfo.getFromProtobuf(part1.getPartKeyInfo()),
omMetadataManager.getDeletedTable().get(part1DeletedKeyName));
ro.getOmKeyInfoList().get(0));
ro = omMetadataManager.getDeletedTable().get(part2DeletedKeyName);
Assert.assertEquals(OmKeyInfo.getFromProtobuf(part2.getPartKeyInfo()),
omMetadataManager.getDeletedTable().get(part2DeletedKeyName));
ro.getOmKeyInfoList().get(0));
}
}