big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11087. Introduce the config to control the refresh interval in RMDelegatedNodeLabelsUpdater. Contributed by Junfan Zhang.

Browse Source
This commit is contained in:
9uapaw 2022-03-22 12:33:17 +01:00
parent 1d5650c4d0
commit 2beb7296fb
3 changed files with 26 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -4423,6 +4423,12 @@ public static boolean areNodeLabelsEnabled(
public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS = public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS =
30 * 60 * 1000; 30 * 60 * 1000;
public static final String RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS =
RM_NODE_LABELS_PROVIDER_PREFIX + "update-newly-registered-nodes-interval-ms";
public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS =
30 * 1000;
@Private @Private
/** /**
* This is a private feature that isn't supposed to be used by end-users. * This is a private feature that isn't supposed to be used by end-users.

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3431,15 +3431,27 @@
<property> <property>
<description> <description>
When "yarn.node-labels.configuration-type" is configured with When "yarn.node-labels.configuration-type" is configured with
"delegated-centralized", then periodically node labels are retrieved "delegated-centralized", then node labels of all nodes
from the node labels provider. This configuration is to define the are updated by periodically retrieving node labels from the
interval. If -1 is configured then node labels are retrieved from provider. If -1 is configured then node labels are retrieved
provider only once for each node after it registers. Defaults to 30 mins. from provider only once for each node after it registers.
Defaults to 30 mins.
</description> </description>
<name>yarn.resourcemanager.node-labels.provider.fetch-interval-ms</name> <name>yarn.resourcemanager.node-labels.provider.fetch-interval-ms</name>
<value>1800000</value> <value>1800000</value>
</property> </property>
<property>
<description>
When "yarn.node-labels.configuration-type" is configured with
"delegated-centralized", then node labels of newly registered
nodes are updated by periodically retrieving node labels from
the provider. Defaults to 30 secs.
</description>
<name>yarn.resourcemanager.node-labels.provider.update-newly-registered-nodes-interval-ms</name>
<value>30000</value>
</property>
<!-- Distributed Node Attributes Configuration --> <!-- Distributed Node Attributes Configuration -->
<property> <property>
<description> <description>

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java
View File

@ -56,7 +56,7 @@ public class RMDelegatedNodeLabelsUpdater extends CompositeService {
private Timer nodeLabelsScheduler; private Timer nodeLabelsScheduler;
// 30 seconds // 30 seconds
@VisibleForTesting @VisibleForTesting
public long nodeLabelsUpdateInterval = 30 * 1000; public long nodeLabelsUpdateInterval;
private Set<NodeId> newlyRegisteredNodes = new HashSet<NodeId>(); private Set<NodeId> newlyRegisteredNodes = new HashSet<NodeId>();
// Lock to protect newlyRegisteredNodes // Lock to protect newlyRegisteredNodes
@ -78,6 +78,9 @@ protected void serviceInit(Configuration conf) throws Exception {
allNodesLabelUpdateInterval = conf.getLong( allNodesLabelUpdateInterval = conf.getLong(
YarnConfiguration.RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS, YarnConfiguration.RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS); YarnConfiguration.DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS);
nodeLabelsUpdateInterval =
conf.getLong(YarnConfiguration.RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS);
rmNodeLabelsMappingProvider = createRMNodeLabelsMappingProvider(conf); rmNodeLabelsMappingProvider = createRMNodeLabelsMappingProvider(conf);
addService(rmNodeLabelsMappingProvider); addService(rmNodeLabelsMappingProvider);
super.serviceInit(conf); super.serviceInit(conf);