From bf9975a1b315942ad34928a980b0fc5544361e36 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Mon, 25 Sep 2023 13:23:02 +0800 Subject: [PATCH] YARN-9586. Need more doc for yarn.federation.policy-manager-params when LoadBasedRouterPolicy is used. (#6085) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../server/router/TestRouterServerUtil.java | 39 +++++++++++++++++++ .../src/test/resources/yarn-site.xml | 4 ++ .../src/site/markdown/Federation.md | 20 ++++++++++ 3 files changed, 63 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java index e82f67d12d..dcf3bd5bc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java @@ -24,6 +24,9 @@ import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo; @@ -33,7 +36,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Map; import static org.apache.hadoop.yarn.server.router.webapp.TestFederationInterceptorREST.getReservationSubmissionRequestInfo; import static org.junit.Assert.assertEquals; @@ -122,4 +128,37 @@ public void testConvertReservationDefinitionEmpty() throws Exception { "definitionInfo Or ReservationRequests is Null.", () -> RouterServerUtil.convertReservationDefinition(definitionInfo3)); } + + @Test + public void testLoadFederationPolicyManager() throws Exception { + + // In this unit test, we have configured the yarn-site.xml file with + // the yarn.federation.policy-manager-params parameter, + // and subsequently, we parse this parameter. + // We have configured two subclusters, SC-1 and SC-2, + // with routerPolicyWeights set to SC-1:0.7 and SC-2:0.3, + // and amrmPolicyWeights set to SC-1:0.6 and SC-2:0.4. + // Additionally, headroomAlpha is set to 1.0. + + YarnConfiguration conf = new YarnConfiguration(); + String defaultPolicyParamString = conf.get(YarnConfiguration.FEDERATION_POLICY_MANAGER_PARAMS, + YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS); + assertNotNull(defaultPolicyParamString); + ByteBuffer defaultPolicyParam = ByteBuffer.wrap( + defaultPolicyParamString.getBytes(StandardCharsets.UTF_8)); + WeightedPolicyInfo policyInfo = WeightedPolicyInfo.fromByteBuffer(defaultPolicyParam); + float headroomAlpha = policyInfo.getHeadroomAlpha(); + Map routerPolicyWeights = policyInfo.getRouterPolicyWeights(); + Map amrmPolicyWeights = policyInfo.getAMRMPolicyWeights(); + + SubClusterIdInfo sc1 = new SubClusterIdInfo("SC-1"); + SubClusterIdInfo sc2 = new SubClusterIdInfo("SC-2"); + + assertEquals(1.0, headroomAlpha, 0.001); + assertEquals(0.7, routerPolicyWeights.get(sc1), 0.001); + assertEquals(0.3, routerPolicyWeights.get(sc2), 0.001); + + assertEquals(0.6, amrmPolicyWeights.get(sc1), 0.001); + assertEquals(0.4, amrmPolicyWeights.get(sc2), 0.001); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml index 84d4171c79..55461680b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml @@ -39,4 +39,8 @@ yarn.resourcemanager.resource-profiles.source-file profiles/sample-profiles-1.json + + yarn.federation.policy-manager-params + {"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index abe2384a39..a613252384 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -235,6 +235,26 @@ SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**. |`yarn.federation.subcluster-resolver.class` | `org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl` | The class used to resolve which subcluster a node belongs to, and which subcluster(s) a rack belongs to. | |`yarn.federation.machine-list` | `` | Path of machine-list file used by `SubClusterResolver`. Each line of the file is a node with sub-cluster and rack information. Below is the example:

node1, subcluster1, rack1
node2, subcluster2, rack1
node3, subcluster3, rack2
node4, subcluster3, rack2 | +- yarn.federation.policy-manager-params + + To configure the `yarn.federation.policy-manager-params` parameter, which represents the weight policy for the default queue, + and where the relevant information will be parsed as `WeightedPolicyInfo`. + + We can use the following JSON format for configuration: + + ```xml + + yarn.federation.policy-manager-params + {"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"} + + ``` + + This JSON configuration allows you to define the weight policy for default queue, where: + + - The `routerPolicyWeights` section specifies the weightings for router policies. For instance, with a weight of `0.3` assigned to `SC-2` and `0.7` assigned to `SC-1`, this configuration will allocate `30%` of submitted application requests to `SC-2` and `70%` to `SC-1`. + - The `amrmPolicyWeights` represents the allocation ratios for Application Master when request containers from different subclusters' RM. For instance, when an AM requests containers, it will request `40%` of the containers from `SC-2` and `60%` of the containers from `SC-1`. + - The `headroomAlpha` used by policies that balance weight-based and load-based considerations in their decisions. For policies that use this parameter, values close to 1 indicate that most of the decision should be based on currently observed headroom from various sub-clusters, values close to zero, indicate that the decision should be mostly based on weights and practically ignore current load. + How to configure the policy-manager --------------------