From 2beb7296fb47e2d40e209c1129bf59f61398b03d Mon Sep 17 00:00:00 2001 From: 9uapaw Date: Tue, 22 Mar 2022 12:33:17 +0100 Subject: [PATCH] YARN-11087. Introduce the config to control the refresh interval in RMDelegatedNodeLabelsUpdater. Contributed by Junfan Zhang. --- .../hadoop/yarn/conf/YarnConfiguration.java | 6 ++++++ .../src/main/resources/yarn-default.xml | 20 +++++++++++++++---- .../RMDelegatedNodeLabelsUpdater.java | 5 ++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 28c96de829..e61b5416e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4423,6 +4423,12 @@ public static boolean areNodeLabelsEnabled( public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS = 30 * 60 * 1000; + public static final String RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS = + RM_NODE_LABELS_PROVIDER_PREFIX + "update-newly-registered-nodes-interval-ms"; + + public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS = + 30 * 1000; + @Private /** * This is a private feature that isn't supposed to be used by end-users. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 39212626ae..1416665a4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3431,15 +3431,27 @@ When "yarn.node-labels.configuration-type" is configured with - "delegated-centralized", then periodically node labels are retrieved - from the node labels provider. This configuration is to define the - interval. If -1 is configured then node labels are retrieved from - provider only once for each node after it registers. Defaults to 30 mins. + "delegated-centralized", then node labels of all nodes + are updated by periodically retrieving node labels from the + provider. If -1 is configured then node labels are retrieved + from provider only once for each node after it registers. + Defaults to 30 mins. yarn.resourcemanager.node-labels.provider.fetch-interval-ms 1800000 + + + When "yarn.node-labels.configuration-type" is configured with + "delegated-centralized", then node labels of newly registered + nodes are updated by periodically retrieving node labels from + the provider. Defaults to 30 secs. + + yarn.resourcemanager.node-labels.provider.update-newly-registered-nodes-interval-ms + 30000 + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java index bb95cf98f4..ba0c430d77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java @@ -56,7 +56,7 @@ public class RMDelegatedNodeLabelsUpdater extends CompositeService { private Timer nodeLabelsScheduler; // 30 seconds @VisibleForTesting - public long nodeLabelsUpdateInterval = 30 * 1000; + public long nodeLabelsUpdateInterval; private Set newlyRegisteredNodes = new HashSet(); // Lock to protect newlyRegisteredNodes @@ -78,6 +78,9 @@ protected void serviceInit(Configuration conf) throws Exception { allNodesLabelUpdateInterval = conf.getLong( YarnConfiguration.RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS); + nodeLabelsUpdateInterval = + conf.getLong(YarnConfiguration.RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS); rmNodeLabelsMappingProvider = createRMNodeLabelsMappingProvider(conf); addService(rmNodeLabelsMappingProvider); super.serviceInit(conf);