YARN-11730. Mark unreported nodes as LOST on RM Startup/HA failover (#7049) Contributed by Arjun Mohnot.

Reviewed-by: Shilun Fan <slfan1989@apache.org>
Signed-off-by: Shilun Fan <slfan1989@apache.org>
This commit is contained in:
Arjun Mohnot 2024-09-23 08:16:20 +05:30 committed by GitHub
parent 81faae6343
commit d8ca2dbe34
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 229 additions and 1 deletions

View File

@ -1277,6 +1277,13 @@ public static boolean isAclEnabled(Configuration conf) {
RM_PREFIX + "nodemanager-graceful-decommission-timeout-secs";
public static final int DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT = 3600;
/**
* Enable/disable tracking of unregistered nodes.
**/
public static final String ENABLE_TRACKING_FOR_UNREGISTERED_NODES =
RM_PREFIX + "enable-tracking-for-unregistered-nodes";
public static final boolean DEFAULT_ENABLE_TRACKING_FOR_UNREGISTERED_NODES = false;
/**
* Period in seconds of the poll timer task inside DecommissioningNodesWatcher
* to identify and take care of DECOMMISSIONING nodes missing regular heart beat.

View File

@ -5810,4 +5810,13 @@
<value>30s</value>
</property>
<property>
<description>
The setting that controls whether the ResourceManager should track the nodes as
lost when they are unregistered and not reported to the RM.
It doesn't account for decommissioned nodes. Default is false.
</description>
<name>yarn.resourcemanager.enable-tracking-for-unregistered-nodes</name>
<value>false</value>
</property>
</configuration>

View File

@ -280,6 +280,7 @@ private void refreshHostsReader(
StringUtils.join(",", hostsReader.getExcludedHosts()) + "}");
handleExcludeNodeList(graceful, timeout);
markUnregisteredNodesAsLost(yarnConf);
}
private void setDecommissionedNMs() {
@ -387,6 +388,115 @@ private void handleExcludeNodeList(boolean graceful, int timeout) {
updateInactiveNodes();
}
/**
* Marks the unregistered nodes as LOST
* if the feature is enabled via a configuration flag.
*
* This method finds nodes that are present in the include list but are not
* registered with the ResourceManager. Such nodes are then marked as LOST.
*
* The steps are as follows:
* 1. Retrieve all hostnames of registered nodes from RM.
* 2. Identify the nodes present in the include list but are not registered
* 3. Remove nodes from the exclude list
* 4. Dispatch LOST events for filtered nodes to mark them as LOST.
*
* @param yarnConf Configuration object that holds the YARN configurations.
*/
private void markUnregisteredNodesAsLost(Configuration yarnConf) {
// Check if tracking unregistered nodes is enabled in the configuration
if (!yarnConf.getBoolean(YarnConfiguration.ENABLE_TRACKING_FOR_UNREGISTERED_NODES,
YarnConfiguration.DEFAULT_ENABLE_TRACKING_FOR_UNREGISTERED_NODES)) {
LOG.debug("Unregistered node tracking is disabled. " +
"Skipping marking unregistered nodes as LOST.");
return;
}
// Set to store all registered hostnames from both active and inactive lists
Set<String> registeredHostNames = gatherRegisteredHostNames();
// Event handler to dispatch LOST events
EventHandler eventHandler = this.rmContext.getDispatcher().getEventHandler();
// Identify nodes that are in the include list but are not registered
// and are not in the exclude list
List<String> nodesToMarkLost = new ArrayList<>();
HostDetails hostDetails = hostsReader.getHostDetails();
Set<String> includes = hostDetails.getIncludedHosts();
Set<String> excludes = hostDetails.getExcludedHosts();
for (String includedNode : includes) {
if (!registeredHostNames.contains(includedNode) && !excludes.contains(includedNode)) {
LOG.info("Lost node: {}", includedNode);
nodesToMarkLost.add(includedNode);
}
}
// Dispatch LOST events for the identified lost nodes
for (String lostNode : nodesToMarkLost) {
dispatchLostEvent(eventHandler, lostNode);
}
// Log successful completion of marking unregistered nodes as LOST
LOG.info("Successfully marked unregistered nodes as LOST");
}
/**
* Gathers all registered hostnames from both active and inactive RMNodes.
*
* @return A set of registered hostnames.
*/
private Set<String> gatherRegisteredHostNames() {
Set<String> registeredHostNames = new HashSet<>();
LOG.info("Getting all the registered hostnames");
// Gather all registered nodes (active) from RM into the set
for (RMNode node : this.rmContext.getRMNodes().values()) {
registeredHostNames.add(node.getHostName());
}
// Gather all inactive nodes from RM into the set
for (RMNode node : this.rmContext.getInactiveRMNodes().values()) {
registeredHostNames.add(node.getHostName());
}
return registeredHostNames;
}
/**
* Dispatches a LOST event for a specified lost node.
*
* @param eventHandler The EventHandler used to dispatch the LOST event.
* @param lostNode The hostname of the lost node for which the event is
* being dispatched.
*/
private void dispatchLostEvent(EventHandler eventHandler, String lostNode) {
// Generate a NodeId for the lost node with a special port -2
NodeId nodeId = createLostNodeId(lostNode);
RMNodeEvent lostEvent = new RMNodeEvent(nodeId, RMNodeEventType.EXPIRE);
RMNodeImpl rmNode = new RMNodeImpl(nodeId, this.rmContext, lostNode, -2, -2,
new UnknownNode(lostNode), Resource.newInstance(0, 0), "unknown");
try {
// Dispatch the LOST event to signal the node is no longer active
eventHandler.handle(lostEvent);
// After successful dispatch, update the node status in RMContext
// Set the node's timestamp for when it became untracked
rmNode.setUntrackedTimeStamp(Time.monotonicNow());
// Add the node to the active and inactive node maps in RMContext
this.rmContext.getRMNodes().put(nodeId, rmNode);
this.rmContext.getInactiveRMNodes().put(nodeId, rmNode);
LOG.info("Successfully dispatched LOST event and deactivated node: {}, Node ID: {}",
lostNode, nodeId);
} catch (Exception e) {
// Log any exception encountered during event dispatch
LOG.error("Error dispatching LOST event for node: {}, Node ID: {} - {}",
lostNode, nodeId, e.getMessage());
}
}
@VisibleForTesting
public int getNodeRemovalCheckInterval() {
return nodeRemovalCheckInterval;
@ -711,6 +821,20 @@ public static NodeId createUnknownNodeId(String host) {
return NodeId.newInstance(host, -1);
}
/**
* Creates a NodeId for a node marked as LOST.
*
* The NodeId combines the hostname with a special port value of -2, indicating
* that the node is lost in the cluster.
*
* @param host The hostname of the lost node.
* @return NodeId Unique identifier for the lost node, with the port set to -2.
*/
public static NodeId createLostNodeId(String host) {
// Create a NodeId with the given host and port -2 to signify the node is lost.
return NodeId.newInstance(host, -2);
}
/**
* A Node instance needed upon startup for populating inactive nodes Map.
* It only knows its hostname/ip.

View File

@ -1603,6 +1603,18 @@ protected void serviceStart() throws Exception {
int port = webApp.port();
WebAppUtils.setRMWebAppPort(conf, port);
}
// Refresh node state before the service startup to reflect the unregistered
// nodemanagers as LOST if the tracking for unregistered nodes flag is enabled.
// For HA setup, refreshNodes is already being called before the active
// transition.
Configuration yarnConf = getConfig();
if (!this.rmContext.isHAEnabled() && yarnConf.getBoolean(
YarnConfiguration.ENABLE_TRACKING_FOR_UNREGISTERED_NODES,
YarnConfiguration.DEFAULT_ENABLE_TRACKING_FOR_UNREGISTERED_NODES)) {
this.rmContext.getNodesListManager().refreshNodes(yarnConf);
}
super.serviceStart();
// Non HA case, start after RM services are started.
@ -1610,7 +1622,7 @@ protected void serviceStart() throws Exception {
transitionToActive();
}
}
protected void doSecureLogin() throws IOException {
InetSocketAddress socAddr = getBindAddress(conf);
SecurityUtil.login(this.conf, YarnConfiguration.RM_KEYTAB,

View File

@ -224,6 +224,9 @@ RMNodeEventType.STARTED, new AddNodeTransition())
.addTransition(NodeState.NEW, NodeState.DECOMMISSIONED,
RMNodeEventType.DECOMMISSION,
new DeactivateNodeTransition(NodeState.DECOMMISSIONED))
.addTransition(NodeState.NEW, NodeState.LOST,
RMNodeEventType.EXPIRE,
new DeactivateNodeTransition(NodeState.LOST))
.addTransition(NodeState.NEW, NodeState.NEW,
RMNodeEventType.FINISHED_CONTAINERS_PULLED_BY_AM,
new AddContainersToBeRemovedFromNMTransition())
@ -958,6 +961,16 @@ public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
if (previousRMNode != null) {
ClusterMetrics.getMetrics().decrDecommisionedNMs();
}
// Check if the node was lost before
NodeId lostNodeId = NodesListManager.createLostNodeId(nodeId.getHost());
RMNode previousRMLostNode = rmNode.context.getInactiveRMNodes().remove(lostNodeId);
if (previousRMLostNode != null) {
// Remove the record of the lost node and update the metrics
rmNode.context.getRMNodes().remove(lostNodeId);
ClusterMetrics.getMetrics().decrNumLostNMs();
}
containers = startEvent.getNMContainerStatuses();
final Resource allocatedResource = Resource.newInstance(
Resources.none());

View File

@ -3303,4 +3303,67 @@ public void testDecommissionWithSelectiveStates() throws Exception {
rm.close();
}
/**
* Test case to verify the behavior of ResourceManager when unregistered nodes
* are marked as 'LOST' and node metrics are correctly updated in the system.
*
* @throws Exception if any unexpected behavior occurs
*/
@Test
public void testMarkUnregisteredNodesAsLost() throws Exception {
// Step 1: Create a Configuration object to hold the settings.
Configuration conf = new Configuration();
// Step 2: Setup the host files.
// Include the following hosts: test_host1, test_host2, test_host3, test_host4
writeToHostsFile(hostFile, "test_host1", "test_host2", "test_host3", "test_host4");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
// Exclude the following host: test_host4
writeToHostsFile(excludeHostFile, "test_host4");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, excludeHostFile.getAbsolutePath());
// Enable tracking for unregistered nodes in the ResourceManager configuration
conf.setBoolean(YarnConfiguration.ENABLE_TRACKING_FOR_UNREGISTERED_NODES, true);
// Step 3: Create a MockRM (ResourceManager) instance to simulate RM behavior
rm = new MockRM(conf);
RMContext rmContext = rm.getRMContext(); // Retrieve the ResourceManager context
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); // Get cluster metrics for nodes
rm.start(); // Start the ResourceManager instance
// Step 4: Register and simulate node activity for "test_host1"
TimeUnit.MILLISECONDS.sleep(50); // Allow some time for event dispatch
MockNM nm1 = rm.registerNode("test_host1:1234", 5120); // Register test_host1 with 5120MB
nm1.nodeHeartbeat(true); // Send heartbeat to simulate the node being alive
TimeUnit.MILLISECONDS.sleep(50); // Allow some time for event processing
// Step 5: Validate that test_host3 is marked as a LOST node
Assert.assertNotNull(clusterMetrics); // Ensure metrics are not null
assertEquals("test_host3 should be a lost NM!",
NodeState.LOST,
rmContext.getInactiveRMNodes().get(
rm.getNodesListManager().createLostNodeId("test_host3")).getState());
// Step 6: Validate node metrics for lost, active, and decommissioned nodes
// Two nodes are lost
assertEquals("There should be 2 Lost NM!", 2, clusterMetrics.getNumLostNMs());
// One node is active
assertEquals("There should be 1 Active NM!", 1, clusterMetrics.getNumActiveNMs());
// One node is decommissioned
assertEquals("There should be 1 Decommissioned NM!", 1,
clusterMetrics.getNumDecommisionedNMs());
// Step 7: Register and simulate node activity for "test_host3"
MockNM nm3 = rm.registerNode("test_host3:5678", 10240); // Register test_host3 with 10240MB
nm3.nodeHeartbeat(true); // Send heartbeat to simulate the node being alive
TimeUnit.MILLISECONDS.sleep(50); // Allow some time for event dispatch and processing
// Step 8: Validate updated node metrics after registering test_host3
assertEquals("There should be 1 Lost NM!", 1,
clusterMetrics.getNumLostNMs()); // Only one node is lost now
assertEquals("There should be 2 Active NM!", 2,
clusterMetrics.getNumActiveNMs()); // Two nodes are now active
}
}