From 43db0cb518375eb767401fa525ea6b5026ed9a8a Mon Sep 17 00:00:00 2001 From: Nanda kumar Date: Wed, 25 Jul 2018 18:11:35 +0530 Subject: [PATCH] HDDS-285. Create a generic Metadata Iterator. Contributed by Bharat Viswanadham. --- .../org/apache/hadoop/utils/LevelDBStore.java | 5 + .../hadoop/utils/LevelDBStoreIterator.java | 64 ++++++ .../hadoop/utils/MetaStoreIterator.java | 39 ++++ .../apache/hadoop/utils/MetadataStore.java | 55 +++++ .../org/apache/hadoop/utils/RocksDBStore.java | 5 + .../hadoop/utils/RocksDBStoreIterator.java | 66 ++++++ .../hadoop/utils/TestMetadataStore.java | 206 +++++++++++------- 7 files changed, 366 insertions(+), 74 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStoreIterator.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetaStoreIterator.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStoreIterator.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStore.java index 13b918015e..ed116a381c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStore.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStore.java @@ -379,4 +379,9 @@ public class LevelDBStore implements MetadataStore { } return result; } + + @Override + public MetaStoreIterator iterator() { + return new LevelDBStoreIterator(db.iterator()); + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStoreIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStoreIterator.java new file mode 100644 index 0000000000..7b62f7ad43 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStoreIterator.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.utils; + +import org.iq80.leveldb.DBIterator; +import java.util.Map; +import java.util.NoSuchElementException; + +import org.apache.hadoop.utils.MetadataStore.KeyValue; + + +/** + * LevelDB store iterator. + */ +public class LevelDBStoreIterator implements MetaStoreIterator { + + + private DBIterator levelDBIterator; + + public LevelDBStoreIterator(DBIterator iterator) { + this.levelDBIterator = iterator; + levelDBIterator.seekToFirst(); + } + + @Override + public boolean hasNext() { + return levelDBIterator.hasNext(); + } + + @Override + public KeyValue next() { + if(levelDBIterator.hasNext()) { + Map.Entry entry = levelDBIterator.next(); + return KeyValue.create(entry.getKey(), entry.getValue()); + } + throw new NoSuchElementException("LevelDB Store has no more elements"); + } + + @Override + public void seekToFirst() { + levelDBIterator.seekToFirst(); + } + + @Override + public void seekToLast() { + levelDBIterator.seekToLast(); + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetaStoreIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetaStoreIterator.java new file mode 100644 index 0000000000..758d19400c --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetaStoreIterator.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.utils; + +import java.util.Iterator; + +/** + * Iterator for MetaDataStore DB. + * @param + */ +interface MetaStoreIterator extends Iterator { + + /** + * seek to first entry. + */ + void seekToFirst(); + + /** + * seek to last entry. + */ + void seekToLast(); + +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStore.java index b90b08f658..7d3bc6ba9a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStore.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStore.java @@ -169,4 +169,59 @@ public interface MetadataStore extends Closeable{ */ void iterate(byte[] from, EntryConsumer consumer) throws IOException; + + /** + * Returns the iterator for this metadata store. + * @return MetaStoreIterator + */ + MetaStoreIterator iterator(); + + /** + * Class used to represent the key and value pair of a db entry. + */ + class KeyValue { + + private final byte[] key; + private final byte[] value; + + /** + * KeyValue Constructor, used to represent a key and value of a db entry. + * @param key + * @param value + */ + private KeyValue(byte[] key, byte[] value) { + this.key = key; + this.value = value; + } + + /** + * Return key. + * @return byte[] + */ + public byte[] getKey() { + byte[] result = new byte[key.length]; + System.arraycopy(key, 0, result, 0, key.length); + return result; + } + + /** + * Return value. + * @return byte[] + */ + public byte[] getValue() { + byte[] result = new byte[value.length]; + System.arraycopy(value, 0, result, 0, value.length); + return result; + } + + /** + * Create a KeyValue pair. + * @param key + * @param value + * @return KeyValue object. + */ + public static KeyValue create(byte[] key, byte[] value) { + return new KeyValue(key, value); + } + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java index 0dfca20a8f..f5f070d242 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java @@ -380,4 +380,9 @@ public class RocksDBStore implements MetadataStore { return statMBeanName; } + @Override + public MetaStoreIterator iterator() { + return new RocksDBStoreIterator(db.newIterator()); + } + } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStoreIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStoreIterator.java new file mode 100644 index 0000000000..6e9b6958da --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStoreIterator.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.utils; + +import org.rocksdb.RocksIterator; + +import java.util.NoSuchElementException; + +import org.apache.hadoop.utils.MetadataStore.KeyValue; + +/** + * RocksDB store iterator. + */ +public class RocksDBStoreIterator implements MetaStoreIterator { + + private RocksIterator rocksDBIterator; + + public RocksDBStoreIterator(RocksIterator iterator) { + this.rocksDBIterator = iterator; + rocksDBIterator.seekToFirst(); + } + + @Override + public boolean hasNext() { + return rocksDBIterator.isValid(); + } + + @Override + public KeyValue next() { + if (rocksDBIterator.isValid()) { + KeyValue value = KeyValue.create(rocksDBIterator.key(), rocksDBIterator + .value()); + rocksDBIterator.next(); + return value; + } + throw new NoSuchElementException("RocksDB Store has no more elements"); + } + + @Override + public void seekToFirst() { + rocksDBIterator.seekToFirst(); + } + + @Override + public void seekToLast() { + rocksDBIterator.seekToLast(); + } + +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java index d697bbfe77..1bce02281b 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java @@ -28,10 +28,10 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.utils.MetadataStore.KeyValue; import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter; import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -48,10 +48,17 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.UUID; + import java.util.concurrent.atomic.AtomicInteger; -import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import static org.junit.runners.Parameterized.Parameters; /** @@ -109,6 +116,58 @@ public class TestMetadataStore { } } + @Test + public void testIterator() throws Exception { + Configuration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); + File dbDir = GenericTestUtils.getRandomizedTestDir(); + MetadataStore dbStore = MetadataStoreBuilder.newBuilder() + .setConf(conf) + .setCreateIfMissing(true) + .setDbFile(dbDir) + .build(); + + //As database is empty, check whether iterator is working as expected or + // not. + MetaStoreIterator metaStoreIterator = dbStore.iterator(); + assertFalse(metaStoreIterator.hasNext()); + try { + metaStoreIterator.next(); + fail("testIterator failed"); + } catch (NoSuchElementException ex) { + GenericTestUtils.assertExceptionContains("Store has no more elements", + ex); + } + + for (int i = 0; i < 10; i++) { + store.put(getBytes("a" + i), getBytes("a-value" + i)); + } + + metaStoreIterator = dbStore.iterator(); + + int i = 0; + while (metaStoreIterator.hasNext()) { + KeyValue val = metaStoreIterator.next(); + assertEquals("a" + i, getString(val.getKey())); + assertEquals("a-value" + i, getString(val.getValue())); + i++; + } + + // As we have iterated all the keys in database, hasNext should return + // false and next() should throw NoSuchElement exception. + + assertFalse(metaStoreIterator.hasNext()); + try { + metaStoreIterator.next(); + fail("testIterator failed"); + } catch (NoSuchElementException ex) { + GenericTestUtils.assertExceptionContains("Store has no more elements", + ex); + } + FileUtils.deleteDirectory(dbDir); + + } + @Test public void testMetaStoreConfigDifferentFromType() throws IOException { @@ -183,17 +242,17 @@ public class TestMetadataStore { public void testGetDelete() throws IOException { for (int i=0; i<10; i++) { byte[] va = store.get(getBytes("a" + i)); - Assert.assertEquals("a-value" + i, getString(va)); + assertEquals("a-value" + i, getString(va)); byte[] vb = store.get(getBytes("b" + i)); - Assert.assertEquals("b-value" + i, getString(vb)); + assertEquals("b-value" + i, getString(vb)); } String keyToDel = "del-" + UUID.randomUUID().toString(); store.put(getBytes(keyToDel), getBytes(keyToDel)); - Assert.assertEquals(keyToDel, getString(store.get(getBytes(keyToDel)))); + assertEquals(keyToDel, getString(store.get(getBytes(keyToDel)))); store.delete(getBytes(keyToDel)); - Assert.assertEquals(null, store.get(getBytes(keyToDel))); + assertEquals(null, store.get(getBytes(keyToDel))); } @Test @@ -228,8 +287,8 @@ public class TestMetadataStore { k = getString(current.getKey()); v = getString(current.getValue()); } - Assert.assertEquals(peekKey, k); - Assert.assertEquals(v, getExpectedValue(peekKey)); + assertEquals(peekKey, k); + assertEquals(v, getExpectedValue(peekKey)); // Look for prev k = null; @@ -240,8 +299,8 @@ public class TestMetadataStore { k = getString(prev.getKey()); v = getString(prev.getValue()); } - Assert.assertEquals(prevKey, k); - Assert.assertEquals(v, getExpectedValue(prevKey)); + assertEquals(prevKey, k); + assertEquals(v, getExpectedValue(prevKey)); // Look for next k = null; @@ -252,8 +311,8 @@ public class TestMetadataStore { k = getString(next.getKey()); v = getString(next.getValue()); } - Assert.assertEquals(nextKey, k); - Assert.assertEquals(v, getExpectedValue(nextKey)); + assertEquals(nextKey, k); + assertEquals(v, getExpectedValue(nextKey)); } @Test @@ -271,9 +330,9 @@ public class TestMetadataStore { return true; }); - Assert.assertFalse(result.isEmpty()); + assertFalse(result.isEmpty()); for (int i=0; i + assertEquals(10, result.size()); + assertTrue(result.stream().allMatch(entry -> new String(entry.getKey()).startsWith("b") )); - Assert.assertEquals(20, filter1.getKeysScannedNum()); - Assert.assertEquals(10, filter1.getKeysHintedNum()); + assertEquals(20, filter1.getKeysScannedNum()); + assertEquals(10, filter1.getKeysHintedNum()); result = store.getRangeKVs(null, 3, filter1); - Assert.assertEquals(3, result.size()); + assertEquals(3, result.size()); result = store.getRangeKVs(getBytes("b3"), 1, filter1); - Assert.assertEquals("b-value3", getString(result.get(0).getValue())); + assertEquals("b-value3", getString(result.get(0).getValue())); // Define a customized filter that filters keys by suffix. // Returns all "*2" entries. MetadataKeyFilter filter2 = (preKey, currentKey, nextKey) -> getString(currentKey).endsWith("2"); result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2); - Assert.assertEquals(2, result.size()); - Assert.assertEquals("a2", getString(result.get(0).getKey())); - Assert.assertEquals("b2", getString(result.get(1).getKey())); + assertEquals(2, result.size()); + assertEquals("a2", getString(result.get(0).getKey())); + assertEquals("b2", getString(result.get(1).getKey())); result = store.getRangeKVs(null, 1, filter2); - Assert.assertEquals(1, result.size()); - Assert.assertEquals("a2", getString(result.get(0).getKey())); + assertEquals(1, result.size()); + assertEquals("a2", getString(result.get(0).getKey())); // Apply multiple filters. result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2); - Assert.assertEquals(1, result.size()); - Assert.assertEquals("b2", getString(result.get(0).getKey())); - Assert.assertEquals("b-value2", getString(result.get(0).getValue())); + assertEquals(1, result.size()); + assertEquals("b2", getString(result.get(0).getKey())); + assertEquals("b-value2", getString(result.get(0).getValue())); // If filter is null, no effect. result = store.getRangeKVs(null, 1, null); - Assert.assertEquals(1, result.size()); - Assert.assertEquals("a0", getString(result.get(0).getKey())); + assertEquals(1, result.size()); + assertEquals("a0", getString(result.get(0).getKey())); } @Test @@ -368,16 +427,16 @@ public class TestMetadataStore { // Suppose to return a2 and b2 List> result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter); - Assert.assertEquals(2, result.size()); - Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey())); - Assert.assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey())); + assertEquals(2, result.size()); + assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey())); + assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey())); // Suppose to return just a2, because when it iterates to a3, // the filter no long matches and it should stop from there. result = store.getSequentialRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter); - Assert.assertEquals(1, result.size()); - Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey())); + assertEquals(1, result.size()); + assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey())); } @Test @@ -385,10 +444,10 @@ public class TestMetadataStore { List> result = null; result = store.getRangeKVs(null, 0); - Assert.assertEquals(0, result.size()); + assertEquals(0, result.size()); result = store.getRangeKVs(null, 1); - Assert.assertEquals(1, result.size()); + assertEquals(1, result.size()); // Count less than zero is invalid. expectedException.expect(IllegalArgumentException.class); @@ -401,7 +460,7 @@ public class TestMetadataStore { // If startKey is invalid, the returned list should be empty. List> kvs = store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH); - Assert.assertEquals(kvs.size(), 0); + assertEquals(kvs.size(), 0); } @Test @@ -421,13 +480,13 @@ public class TestMetadataStore { dbStore.put(getBytes("key1"), getBytes("value1")); dbStore.put(getBytes("key2"), getBytes("value2")); - Assert.assertFalse(dbStore.isEmpty()); - Assert.assertTrue(dbDir.exists()); - Assert.assertTrue(dbDir.listFiles().length > 0); + assertFalse(dbStore.isEmpty()); + assertTrue(dbDir.exists()); + assertTrue(dbDir.listFiles().length > 0); dbStore.destroy(); - Assert.assertFalse(dbDir.exists()); + assertFalse(dbDir.exists()); } @Test @@ -469,7 +528,7 @@ public class TestMetadataStore { return it.hasNext() && it.next().equals(getString(key)); }); - Assert.assertEquals(8, count.get()); + assertEquals(8, count.get()); } @Test @@ -482,52 +541,51 @@ public class TestMetadataStore { } catch (IllegalArgumentException e) { exception = e; } - Assert.assertTrue( - exception.getMessage().contains("KeyPrefix: b already rejected")); + assertTrue(exception.getMessage().contains("KeyPrefix: b already " + + "rejected")); try { new KeyPrefixFilter().addFilter("b0").addFilter("b", true); } catch (IllegalArgumentException e) { exception = e; } - Assert.assertTrue( - exception.getMessage().contains("KeyPrefix: b already accepted")); + assertTrue(exception.getMessage().contains("KeyPrefix: b already " + + "accepted")); try { new KeyPrefixFilter().addFilter("b", true).addFilter("b0"); } catch (IllegalArgumentException e) { exception = e; } - Assert.assertTrue( - exception.getMessage().contains("KeyPrefix: b0 already rejected")); + assertTrue(exception.getMessage().contains("KeyPrefix: b0 already " + + "rejected")); try { new KeyPrefixFilter().addFilter("b").addFilter("b0", true); } catch (IllegalArgumentException e) { exception = e; } - Assert.assertTrue( - exception.getMessage().contains("KeyPrefix: b0 already accepted")); + assertTrue(exception.getMessage().contains("KeyPrefix: b0 already " + + "accepted")); MetadataKeyFilter filter1 = new KeyPrefixFilter(true) .addFilter("a0") .addFilter("a1") .addFilter("b", true); result = store.getRangeKVs(null, 100, filter1); - Assert.assertEquals(2, result.size()); - Assert.assertTrue(result.stream() - .anyMatch(entry -> new String(entry.getKey()).startsWith("a0")) - && result.stream() - .anyMatch(entry -> new String(entry.getKey()).startsWith("a1"))); + assertEquals(2, result.size()); + assertTrue(result.stream().anyMatch(entry -> new String(entry.getKey()) + .startsWith("a0")) && result.stream().anyMatch(entry -> new String( + entry.getKey()).startsWith("a1"))); filter1 = new KeyPrefixFilter(true).addFilter("b", true); result = store.getRangeKVs(null, 100, filter1); - Assert.assertEquals(0, result.size()); + assertEquals(0, result.size()); filter1 = new KeyPrefixFilter().addFilter("b", true); result = store.getRangeKVs(null, 100, filter1); - Assert.assertEquals(10, result.size()); - Assert.assertTrue(result.stream() - .allMatch(entry -> new String(entry.getKey()).startsWith("a"))); + assertEquals(10, result.size()); + assertTrue(result.stream().allMatch(entry -> new String(entry.getKey()) + .startsWith("a"))); } }