From adba43d1540019c105c24e97849647d42e130d04 Mon Sep 17 00:00:00 2001 From: zeekling Date: Sat, 12 Oct 2024 00:25:56 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Router=E8=AF=A6=E8=A7=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hdfs/router启动详解.md | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/hdfs/router启动详解.md b/hdfs/router启动详解.md index d2f4b19..898c747 100644 --- a/hdfs/router启动详解.md +++ b/hdfs/router启动详解.md @@ -9,7 +9,9 @@ # 子模块 -## State Store模块 +## 一、State Store模块 + +当前模块主要用于保存Router状态信息,提供了文件系统(HDFS、本地)、Mysql、Zookeeper。个人觉得一般使用Zookeeper比较合理。 ### 初始化 @@ -149,7 +151,7 @@ MySQLStateStoreHikariDataSourceConnectionFactory(Configuration conf) { ##### initRecordStorage -zai StateStoreMySQLImpl当中,每个state store 对应一张表。建表语句如下: +在StateStoreMySQLImpl当中,每个state store 对应一张表。建表语句如下: ```sql CREATE TABLE ( recordKey VARCHAR (255) NOT NULL, @@ -163,10 +165,43 @@ CREATE TABLE ( ##### initDriver +对于StateStoreZooKeeperImpl,initDriver中主要是建立zk连接。核心代码如下: +```java +this.zkManager = new ZKCuratorManager(conf); +this.zkManager.start(zkHostPort); +this.zkAcl = ZKCuratorManager.getZKAcls(conf); +``` +对于异步模式,会创建线程池,用于异步保存状态。 +```java +ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("StateStore ZK Client-%d") + .build(); +this.executorService = new ThreadPoolExecutor(numThreads, numThreads, + 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), threadFactory); +LOG.info("Init StateStoreZookeeperImpl by async mode with {} threads.", numThreads); +``` + ##### initRecordStorage +当前函数的实现比较简单,主要是在zk上面创建状态保存的目录。 +```java +try { + String checkPath = getNodePath(baseZNode, className); + zkManager.createRootDirRecursively(checkPath, zkAcl); + return true; +} catch (Exception e) { + LOG.error("Cannot initialize ZK node for {}: {}", + className, e.getMessage()); + return false; +} +``` + +## 二、ActiveNamenodeResolver +具体实现有配置项dfs.federation.router.namenode.resolver.client.class指定。默认为MembershipNamenodeResolver.class。目前最新的也只支持这一种。 + +### MembershipNamenodeResolver + -## ActiveNamenodeResolver ## subclusterResolver