big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12546. Ozone: DB listing operation performance improvement. Contributed by Weiwei Yang.

Browse Source
This commit is contained in:
Chen Liang 2017-10-09 14:52:50 -07:00 committed by Owen O'Malley
parent b05ad0b315
commit ae2b77a0e5
4 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
View File

@ -180,7 +180,7 @@ public List<KeyData> listKey(
DFSUtil.string2Bytes(startKey);
MetadataKeyFilter prefixFilter = new KeyPrefixFilter(prefix);
List<Map.Entry<byte[], byte[]>> range =
db.getRangeKVs(startKeyInBytes, count, prefixFilter);
db.getSequentialRangeKVs(startKeyInBytes, count, prefixFilter);
for (Map.Entry<byte[], byte[]> entry : range) {
String keyName = KeyUtils.getKeyName(entry.getKey());
KeyData value = KeyUtils.getKeyData(entry.getValue());

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
View File

@ -175,7 +175,7 @@ public BackgroundTaskResult call() throws Exception {
KeyPrefixFilter filter = new KeyPrefixFilter(
OzoneConsts.DELETING_KEY_PREFIX);
List<Map.Entry<byte[], byte[]>> toDeleteBlocks =
meta.getRangeKVs(null, blockLimitPerTask, filter);
meta.getSequentialRangeKVs(null, blockLimitPerTask, filter);
if (toDeleteBlocks.isEmpty()) {
LOG.debug("No under deletion block found in container : {}",
containerData.getContainerName());

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
View File

@ -347,13 +347,13 @@ public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
if (!Strings.isNullOrEmpty(startKey)) {
//Since we are excluding start key from the result,
// the maxNumOfBuckets is incremented.
rangeResult = store.getRangeKVs(
rangeResult = store.getSequentialRangeKVs(
getDBKeyBytes(volumeName, bucketName, startKey),
maxKeys + 1, filter);
//Remove start key from result.
rangeResult.remove(0);
} else {
rangeResult = store.getRangeKVs(null, maxKeys, filter);
rangeResult = store.getSequentialRangeKVs(null, maxKeys, filter);
}
for (Map.Entry<byte[], byte[]> entry : rangeResult) {
@ -440,7 +440,7 @@ private VolumeList getAllVolumes() throws IOException {
// it should be fine to scan all users in db and return us a
// list of volume names in string per user.
List<Map.Entry<byte[], byte[]>> rangeKVs = store
.getRangeKVs(null, Integer.MAX_VALUE, filter);
.getSequentialRangeKVs(null, Integer.MAX_VALUE, filter);
VolumeList.Builder builder = VolumeList.newBuilder();
for (Map.Entry<byte[], byte[]> entry : rangeKVs) {
@ -458,7 +458,7 @@ public List<BlockGroup> getPendingDeletionKeys(final int count)
final MetadataKeyFilter deletingKeyFilter =
new KeyPrefixFilter(DELETING_KEY_PREFIX);
List<Map.Entry<byte[], byte[]>> rangeResult =
store.getRangeKVs(null, count, deletingKeyFilter);
store.getSequentialRangeKVs(null, count, deletingKeyFilter);
for (Map.Entry<byte[], byte[]> entry : rangeResult) {
KsmKeyInfo info =
KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(entry.getValue()));

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
View File

@ -145,7 +145,7 @@ public List<Pipeline> listContainer(String startName, String prefixName,
byte[] startKey = startName == null ? null : DFSUtil.string2Bytes(
startName);
List<Map.Entry<byte[], byte[]>> range =
containerStore.getRangeKVs(startKey, count, prefixFilter);
containerStore.getSequentialRangeKVs(startKey, count, prefixFilter);
// Transform the values into the pipelines.
// TODO: return list of ContainerInfo instead of pipelines.