From 35d77a6359c27aab6dda63b846811e5415d20867 Mon Sep 17 00:00:00 2001
From: slfan1989 <55643692+slfan1989@users.noreply.github.com>
Date: Wed, 21 Jun 2023 01:23:11 +0800
Subject: [PATCH] YARN-11505. [Federation] Add Steps To Set up a Test Cluster.
(#5751)
---
.../src/site/markdown/Federation.md | 264 ++++++++++++++++++
1 file changed, 264 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index c5fcef8b2e..1209eb95c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -385,3 +385,267 @@ The output from this particular example job should be something like:
The state of the job can also be tracked on the Router Web UI at `routerhost:8089`.
Note that no change in the code or recompilation of the input jar was required to use federation. Also, the output of this job is the exact same as it would be when run without federation. Also, in order to get the full benefit of federation, use a large enough number of mappers such that more than one cluster is required. That number happens to be 16 in the case of the above example.
+
+How to build a Test Federation Cluster
+--------------------
+
+The purpose of this document is to help users quickly set up a testing environment for YARN Federation. With this testing environment, users can utilize the core functionality of YARN Federation. This is the simplest test cluster setup (based on Linux) with only essential configurations (YARN non-HA mode). We require 3 machines, and each machine should have at least <4C, 8GB> of resources. We only cover YARN configuration in this document. For information on configuring HDFS and ZooKeeper, please refer to other documentation sources.
+
+Test Environment Description:
+- We need to build a HDFS test environment, this part can refer to HDFS documentation. [HDFS SingleCluster](../../hadoop-project-dist/hadoop-common/SingleCluster.html)
+- We need two YARN clusters, each YARN cluster has one RM and one NM, The RM and NM on the same node.
+- We need one ZK cluster(We only need one ZooKeeper node.), this part can refer to Zookeeper documentation. [ZookeeperStarted](https://zookeeper.apache.org/doc/current/zookeeperStarted.html)
+- We need one Router and one Client.
+
+Example of Machine-Role Mapping(Exclude HDFS):
+
+| Machine | Role |
+|:----------|:--------------|
+| Machine A | RM1\NM1\ZK1 |
+| Machine B | RM2\NM2 |
+| Machine C | Router\Client |
+
+### YARN-1(ClusterTest-Yarn1)
+
+#### RM-1
+
+- For the ResourceManager, we need to configure the following option:
+
+```xml
+
+
+
+ yarn.resourcemanager.cluster-id
+ ClusterTest-Yarn1
+
+
+
+
+ yarn.resourcemanager.scheduler.class
+ org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
+
+
+
+
+ yarn.scheduler.fair.allocation.file
+ /path/fair-scheduler.xml
+
+
+
+
+ yarn.federation.enabled
+ true
+
+
+
+
+ yarn.federation.state-store.class
+ org.apache.hadoop.yarn.server.federation.store.impl.ZookeeperFederationStateStore
+
+
+
+
+ hadoop.zk.address
+ zkHost:zkPort
+
+
+```
+
+- Start RM
+
+```
+$HADOOP_HOME/bin/yarn --daemon start resourcemanager
+```
+
+#### NM-1
+
+- For the NodeManager, we need to configure the following option:
+
+```xml
+
+
+ yarn.resourcemanager.cluster-id
+ ClusterTest-Yarn1
+
+
+
+
+ yarn.nodemanager.local-dirs
+ path/local
+
+
+
+
+ yarn.nodemanager.log-dirs
+ path/logdir
+
+
+
+
+ yarn.federation.enabled
+ true
+
+
+
+
+ yarn.federation.failover.enabled
+ false
+
+
+
+
+ yarn.federation.non-ha.enabled
+ true
+
+
+
+
+ yarn.federation.state-store.class
+ org.apache.hadoop.yarn.server.federation.store.impl.ZookeeperFederationStateStore
+
+
+
+
+ hadoop.zk.address
+ zkHost:zkPort
+
+
+
+
+ yarn.nodemanager.amrmproxy.enabled
+ true
+
+
+
+
+ yarn.nodemanager.amrmproxy.interceptor-class.pipeline
+ org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor
+
+```
+
+- Start NM
+
+```
+$HADOOP_HOME/bin/yarn --daemon start nodemanager
+```
+
+### YARN-2(ClusterTest-Yarn2)
+
+#### RM-2
+
+The RM of the `YARN-2` cluster is configured the same as the RM of `YARN-1` except for the `cluster-id`
+
+```xml
+
+ yarn.resourcemanager.cluster-id
+ ClusterTest-Yarn2
+
+```
+
+#### NM-2
+
+The NM of the `YARN-2` cluster is configured the same as the RM of `YARN-1` except for the `cluster-id`
+
+```xml
+
+ yarn.resourcemanager.cluster-id
+ ClusterTest-Yarn2
+
+```
+
+After we have finished configuring the `YARN-2` cluster, we can proceed with starting the `YARN-2` cluster.
+
+### Router
+
+- For the Router, we need to configure the following option:
+
+```xml
+
+
+ yarn.federation.enabled
+ true
+
+
+
+
+ yarn.federation.state-store.class
+ org.apache.hadoop.yarn.server.federation.store.impl.ZookeeperFederationStateStore
+
+
+
+
+ hadoop.zk.address
+ zkHost:zkPort
+
+
+
+
+ yarn.router.clientrm.interceptor-class.pipeline
+ org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor
+
+
+
+
+ yarn.router.webapp.interceptor-class.pipeline
+ org.apache.hadoop.yarn.server.router.webapp.FederationInterceptorREST
+
+
+
+
+ yarn.router.rmadmin.interceptor-class.pipeline
+ org.apache.hadoop.yarn.server.router.rmadmin.FederationRMAdminInterceptor
+
+```
+
+- Start Router
+
+```
+$HADOOP_HOME/bin/yarn --daemon start router
+```
+
+### Yarn-Client
+
+- For the Yarn-Client, we need to configure the following option:
+
+```xml
+
+
+
+ yarn.federation.enabled
+ true
+
+
+
+
+ yarn.federation.failover.enabled
+ false
+
+
+
+
+ yarn.resourcemanager.address
+ router-1-Host:8050
+
+
+
+
+ yarn.resourcemanager.admin.address
+ router-1-Host:8052
+
+
+
+
+ yarn.resourcemanager.scheduler.address
+ localhost:8049
+
+```
\ No newline at end of file