big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12524. Ozone: Record number of keys scanned and hinted for getRangeKVs call. Contributed by Weiwei Yang.

Browse Source
This commit is contained in:
Weiwei Yang 2017-09-26 19:22:18 +08:00 committed by Owen O'Malley
parent 712bd70ac7
commit d303b7f40f
4 changed files with 59 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
View File

@ -353,11 +353,23 @@ private List<Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
if (dbIter != null) {
dbIter.close();
}
long end = System.currentTimeMillis();
long timeConsumed = end - start;
if (LOG.isDebugEnabled()) {
LOG.debug("Time consumed for getRangeKVs() is {},"
+ " result length is {}.", timeConsumed, result.size());
if (filters != null) {
for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
int scanned = filter.getKeysScannedNum();
int hinted = filter.getKeysHintedNum();
if (scanned > 0 || hinted > 0) {
LOG.debug(
"getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
filter.getClass().getSimpleName(), filter.getKeysScannedNum(),
filter.getKeysHintedNum());
}
}
}
long end = System.currentTimeMillis();
long timeConsumed = end - start;
LOG.debug("Time consumed for getRangeKVs() is {}ms,"
+ " result length is {}.", timeConsumed, result.size());
}
}
return result;

32
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
View File

@ -38,6 +38,14 @@ public interface MetadataKeyFilter {
* @return true if a certain condition satisfied, return false otherwise.
*/
boolean filterKey(byte[] preKey, byte[] currentKey, byte[] nextKey);
default int getKeysScannedNum() {
return 0;
}
default int getKeysHintedNum() {
return 0;
}
}
/**
@ -47,19 +55,37 @@ public interface MetadataKeyFilter {
public static class KeyPrefixFilter implements MetadataKeyFilter {
private String keyPrefix = null;
private int keysScanned = 0;
private int keysHinted = 0;
public KeyPrefixFilter(String keyPrefix) {
this.keyPrefix = keyPrefix;
}
@Override public boolean filterKey(byte[] preKey, byte[] currentKey,
@Override
public boolean filterKey(byte[] preKey, byte[] currentKey,
byte[] nextKey) {
keysScanned++;
if (Strings.isNullOrEmpty(keyPrefix)) {
return true;
} else {
return currentKey != null &&
DFSUtil.bytes2String(currentKey).startsWith(keyPrefix);
if (currentKey != null &&
DFSUtil.bytes2String(currentKey).startsWith(keyPrefix)) {
keysHinted++;
return true;
}
return false;
}
}
@Override
public int getKeysScannedNum() {
return keysScanned;
}
@Override
public int getKeysHintedNum() {
return keysHinted;
}
}
}

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
View File

@ -201,11 +201,22 @@ private List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
long end = System.currentTimeMillis();
long timeConsumed = end - start;
if (LOG.isDebugEnabled()) {
if (filters != null) {
for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
int scanned = filter.getKeysScannedNum();
int hinted = filter.getKeysHintedNum();
if (scanned > 0 || hinted > 0) {
LOG.debug(
"getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
filter.getClass().getSimpleName(), filter.getKeysScannedNum(),
filter.getKeysHintedNum());
}
}
}
LOG.debug("Time consumed for getRangeKVs() is {}ms,"
+ " result length is {}.", timeConsumed, result.size());
}
}
return result;
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
View File

@ -265,6 +265,8 @@ public void testGetRangeKVs() throws IOException {
Assert.assertTrue(result.stream().allMatch(entry ->
new String(entry.getKey()).startsWith("b")
));
Assert.assertEquals(20, filter1.getKeysScannedNum());
Assert.assertEquals(10, filter1.getKeysHintedNum());
result = store.getRangeKVs(null, 3, filter1);
Assert.assertEquals(3, result.size());
result = store.getRangeKVs(getBytes("b3"), 1, filter1);