From d50a743015ff1ca036d437302486b0fd0c09cb46 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Wed, 26 Jul 2017 12:59:25 +0800 Subject: [PATCH] HDFS-12155. Ozone : add RocksDB support to DEBUG CLI. Contributed by Chen Liang. --- .../apache/hadoop/ozone/scm/cli/SQLCLI.java | 15 +++--- .../hadoop/ozone/ksm/TestKSMSQLCli.java | 24 +++++++-- .../hadoop/ozone/scm/TestContainerSQLCli.java | 54 +++++++++++++------ 3 files changed, 66 insertions(+), 27 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java index 59fe2ae511..30a2d22f45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.OzoneAclInfo; import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.BucketInfo; import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo; @@ -75,6 +76,7 @@ public class SQLCLI extends Configured implements Tool { private Options options; private BasicParser parser; private final Charset encoding = Charset.forName("UTF-8"); + private final OzoneConfiguration conf; // for container.db private static final String CREATE_CONTAINER_INFO = @@ -199,9 +201,10 @@ public class SQLCLI extends Configured implements Tool { private static final Logger LOG = LoggerFactory.getLogger(SQLCLI.class); - public SQLCLI() { + public SQLCLI(OzoneConfiguration conf) { this.options = getOptions(); this.parser = new BasicParser(); + this.conf = conf; } @SuppressWarnings("static-access") @@ -468,7 +471,7 @@ private void convertContainerDB(Path dbPath, Path outPath) LOG.info("Create tables for sql container db."); File dbFile = dbPath.toFile(); try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setDbFile(dbFile).build(); + .setConf(conf).setDbFile(dbFile).build(); Connection conn = connectDB(outPath.toString())) { executeSQL(conn, CREATE_CONTAINER_INFO); executeSQL(conn, CREATE_CONTAINER_MEMBERS); @@ -547,7 +550,7 @@ private void convertBlockDB(Path dbPath, Path outPath) throws Exception { LOG.info("Create tables for sql block db."); File dbFile = dbPath.toFile(); try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setDbFile(dbFile).build(); + .setConf(conf).setDbFile(dbFile).build(); Connection conn = connectDB(outPath.toString())) { executeSQL(conn, CREATE_BLOCK_CONTAINER); @@ -594,7 +597,7 @@ private void convertNodePoolDB(Path dbPath, Path outPath) throws Exception { LOG.info("Create table for sql node pool db."); File dbFile = dbPath.toFile(); try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setDbFile(dbFile).build(); + .setConf(conf).setDbFile(dbFile).build(); Connection conn = connectDB(outPath.toString())) { executeSQL(conn, CREATE_NODE_POOL); executeSQL(conn, CREATE_DATANODE_INFO); @@ -645,7 +648,7 @@ private void convertOpenContainerDB(Path dbPath, Path outPath) LOG.info("Create table for open container db."); File dbFile = dbPath.toFile(); try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setDbFile(dbFile).build(); + .setConf(conf).setDbFile(dbFile).build(); Connection conn = connectDB(outPath.toString())) { executeSQL(conn, CREATE_OPEN_CONTAINER); @@ -671,7 +674,7 @@ private CommandLine parseArgs(String[] argv) } public static void main(String[] args) { - Tool shell = new SQLCLI(); + Tool shell = new SQLCLI(new OzoneConfiguration()); int res = 0; try { ToolRunner.run(shell, args); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java index 97979ed8ae..6a2b0f6ced 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java @@ -30,9 +30,9 @@ import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.IOException; import java.io.OutputStream; @@ -44,6 +44,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -57,6 +58,7 @@ /** * This class tests the CLI that transforms ksm.db into SQLite DB files. */ +@RunWith(Parameterized.class) public class TestKSMSQLCli { private static MiniOzoneCluster cluster = null; private static StorageHandler storageHandler; @@ -76,8 +78,19 @@ public class TestKSMSQLCli { private static String keyName2 = "key2"; private static String keyName3 = "key3"; - @Rule - public ExpectedException exception = ExpectedException.none(); + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][] { + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB} + }); + } + + private static String metaStoreType; + + public TestKSMSQLCli(String type) { + metaStoreType = type; + } /** * Create a MiniDFSCluster for testing. @@ -142,7 +155,8 @@ public static void setup() throws Exception { @Before public void init() throws Exception { - cli = new SQLCLI(); + conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType); + cli = new SQLCLI(conf); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java index 1e6f5f7cb9..6ffbb4ba1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java @@ -33,9 +33,11 @@ import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock; import org.apache.hadoop.scm.container.common.helpers.Pipeline; import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.nio.file.Files; import java.nio.file.Paths; @@ -45,6 +47,8 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -59,27 +63,43 @@ /** * This class tests the CLI that transforms container into SQLite DB files. */ +@RunWith(Parameterized.class) public class TestContainerSQLCli { + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][] { + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB} + }); + } + + private static String metaStoreType; + + public TestContainerSQLCli(String type) { + metaStoreType = type; + } + private static SQLCLI cli; - private static MiniOzoneCluster cluster; - private static OzoneConfiguration conf; - private static StorageContainerLocationProtocolClientSideTranslatorPB + private MiniOzoneCluster cluster; + private OzoneConfiguration conf; + private StorageContainerLocationProtocolClientSideTranslatorPB storageContainerLocationClient; - private static ContainerMapping mapping; - private static NodeManager nodeManager; - private static BlockManagerImpl blockManager; + private ContainerMapping mapping; + private NodeManager nodeManager; + private BlockManagerImpl blockManager; - private static Pipeline pipeline1; - private static Pipeline pipeline2; + private Pipeline pipeline1; + private Pipeline pipeline2; - private static HashMap blockContainerMap; + private HashMap blockContainerMap; private final static long DEFAULT_BLOCK_SIZE = 4 * KB; - @BeforeClass - public static void init() throws Exception { + @Before + public void setup() throws Exception { long datanodeCapacities = 3 * OzoneConsts.TB; blockContainerMap = new HashMap<>(); @@ -138,11 +158,13 @@ public static void init() throws Exception { mapping.close(); nodeManager.close(); - cli = new SQLCLI(); + conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType); + cli = new SQLCLI(conf); + } - @AfterClass - public static void shutdown() throws InterruptedException { + @After + public void shutdown() throws InterruptedException { IOUtils.cleanup(null, storageContainerLocationClient, cluster); }