big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11087. Introduce the config to control the refresh interval in RMDelegatedNodeLabelsUpdater. Contributed by Junfan Zhang.

Browse Source
This commit is contained in:
9uapaw 2022-03-22 12:33:17 +01:00
parent 1d5650c4d0
commit 2beb7296fb
3 changed files with 26 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -4423,6 +4423,12 @@ public static boolean areNodeLabelsEnabled(
public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS =
30 * 60 * 1000;
public static final String RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS =
RM_NODE_LABELS_PROVIDER_PREFIX + "update-newly-registered-nodes-interval-ms";
public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS =
30 * 1000;
@Private
/**
* This is a private feature that isn't supposed to be used by end-users.

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3431,15 +3431,27 @@
<property>
<description>
When "yarn.node-labels.configuration-type" is configured with
"delegated-centralized", then periodically node labels are retrieved
from the node labels provider. This configuration is to define the
interval. If -1 is configured then node labels are retrieved from
provider only once for each node after it registers. Defaults to 30 mins.
"delegated-centralized", then node labels of all nodes
are updated by periodically retrieving node labels from the
provider. If -1 is configured then node labels are retrieved
from provider only once for each node after it registers.
Defaults to 30 mins.
</description>
<name>yarn.resourcemanager.node-labels.provider.fetch-interval-ms</name>
<value>1800000</value>
</property>
<property>
<description>
When "yarn.node-labels.configuration-type" is configured with
"delegated-centralized", then node labels of newly registered
nodes are updated by periodically retrieving node labels from
the provider. Defaults to 30 secs.
</description>
<name>yarn.resourcemanager.node-labels.provider.update-newly-registered-nodes-interval-ms</name>
<value>30000</value>
</property>
<!-- Distributed Node Attributes Configuration -->
<property>
<description>

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java
View File

@ -56,7 +56,7 @@ public class RMDelegatedNodeLabelsUpdater extends CompositeService {
private Timer nodeLabelsScheduler;
// 30 seconds
@VisibleForTesting
public long nodeLabelsUpdateInterval = 30 * 1000;
public long nodeLabelsUpdateInterval;
private Set<NodeId> newlyRegisteredNodes = new HashSet<NodeId>();
// Lock to protect newlyRegisteredNodes
@ -78,6 +78,9 @@ protected void serviceInit(Configuration conf) throws Exception {
allNodesLabelUpdateInterval = conf.getLong(
YarnConfiguration.RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS);
nodeLabelsUpdateInterval =
conf.getLong(YarnConfiguration.RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS);
rmNodeLabelsMappingProvider = createRMNodeLabelsMappingProvider(conf);
addService(rmNodeLabelsMappingProvider);
super.serviceInit(conf);