big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10300: appMasterHost not set in RM ApplicationSummary when AM fails before first heartbeat. Contributed by Eric Badger (ebadger).

Browse Source
This commit is contained in:
Eric E Payne 2020-06-09 18:43:16 +00:00
parent ac5d899d40
commit 56247db302
3 changed files with 62 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -26,6 +26,8 @@
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@ -190,7 +192,16 @@ public static SummaryBuilder createAppSummary(RMApp app) {
RMAppAttempt attempt = app.getCurrentAppAttempt();
if (attempt != null) {
trackingUrl = attempt.getTrackingUrl();
host = attempt.getHost();
Container masterContainer = attempt.getMasterContainer();
if (masterContainer != null) {
NodeId nodeId = masterContainer.getNodeId();
if (nodeId != null) {
String amHost = nodeId.getHost();
if (amHost != null) {
host = amHost;
}
}
}
}
RMAppMetrics metrics = app.getRMAppMetrics();
SummaryBuilder summary = new SummaryBuilder()

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -22,7 +22,10 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@ -972,6 +975,17 @@ public void testEscapeApplicationSummary() {
when(app.getSubmitTime()).thenReturn(1000L);
when(app.getLaunchTime()).thenReturn(2000L);
when(app.getApplicationTags()).thenReturn(Sets.newHashSet("tag2", "tag1"));
RMAppAttempt mockRMAppAttempt = mock(RMAppAttempt.class);
Container mockContainer = mock(Container.class);
NodeId mockNodeId = mock(NodeId.class);
String host = "127.0.0.1";
when(mockNodeId.getHost()).thenReturn(host);
when(mockContainer.getNodeId()).thenReturn(mockNodeId);
when(mockRMAppAttempt.getMasterContainer()).thenReturn(mockContainer);
when(app.getCurrentAppAttempt()).thenReturn(mockRMAppAttempt);
Map<String, Long> resourceSecondsMap = new HashMap<>();
resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L);
resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L);
@ -993,6 +1007,7 @@ public void testEscapeApplicationSummary() {
assertTrue(msg.contains("Multiline" + escaped +"AppName"));
assertTrue(msg.contains("Multiline" + escaped +"UserName"));
assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
assertTrue(msg.contains("appMasterHost=" + host));
assertTrue(msg.contains("submitTime=1000"));
assertTrue(msg.contains("launchTime=2000"));
assertTrue(msg.contains("memorySeconds=16384"));

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
View File

@ -453,6 +453,41 @@ public void testSetupTokensWithHTTPS() throws Exception {
testSetupTokens(true, conf);
}
@Test
public void testAMMasterContainerHost() throws Exception {
//Test that masterContainer and its associated host are
//set before the AM is even launched.
MockRM rm = new MockRM();
rm.start();
String host = "127.0.0.1";
String port = "1234";
MockNM nm1 = rm.registerNode(host + ":" + port, 5120);
RMApp app = MockRMAppSubmitter.submitWithMemory(2000, rm);
// kick the scheduling
nm1.nodeHeartbeat(true);
RMAppAttempt attempt = app.getCurrentAppAttempt();
try {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override public Boolean get() {
return attempt.getMasterContainer() != null;
}
}, 10, 200 * 100);
} catch (TimeoutException e) {
fail("timed out while waiting for AM Launch to happen.");
}
Assert.assertEquals(
app.getCurrentAppAttempt().getMasterContainer().getNodeId().getHost(),
host);
//send kill before launch
rm.killApp(app.getApplicationId());
rm.waitForState(app.getApplicationId(), RMAppState.KILLED);
rm.stop();
}
private void testSetupTokens(boolean https, YarnConfiguration conf)
throws Exception {
MockRM rm = new MockRM(conf);