big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6149. Allow port range to be specified while starting NM Timeline collector manager. Contributed by Abhishek Modi.

Browse Source
This commit is contained in:
Rohith Sharma K S 2019-01-04 14:07:02 +05:30
parent 573b158791
commit 8c6978c3ba
3 changed files with 21 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -3868,6 +3868,10 @@ public static boolean areNodeLabelsEnabled(
public static final String TIMELINE_SERVICE_COLLECTOR_BIND_HOST =
TIMELINE_SERVICE_COLLECTOR_PREFIX + "bind-host";
@Private
public static final String TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES =
TIMELINE_SERVICE_COLLECTOR_PREFIX + "bind-port-ranges";
@Private
public static final String TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS =
TIMELINE_SERVICE_COLLECTOR_PREFIX + "webapp.address";

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
View File

@ -58,6 +58,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class on the NodeManager side that manages adding and removing collectors and
* their lifecycle. Also instantiates the per-node collector webapp.
@ -280,14 +281,21 @@ private void startWebApp() {
String bindAddress = null;
String host =
conf.getTrimmed(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_HOST);
Configuration.IntegerRanges portRanges = conf.getRange(
YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES, "");
int startPort = 0;
if (portRanges != null && !portRanges.isEmpty()) {
startPort = portRanges.getRangeStart();
}
if (host == null || host.isEmpty()) {
// if collector bind-host is not set, fall back to
// timeline-service.bind-host to maintain compatibility
bindAddress =
conf.get(YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST)
+ ":" + startPort;
} else {
bindAddress = host + ":0";
bindAddress = host + ":" + startPort;
}
try {
@ -297,6 +305,9 @@ private void startWebApp() {
.addEndpoint(URI.create(
(YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
bindAddress));
if (portRanges != null && !portRanges.isEmpty()) {
builder.setPortRanges(portRanges);
}
if (YarnConfiguration.useHttps(conf)) {
builder = WebAppUtils.loadSslConfiguration(builder, conf);
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
View File

@ -60,6 +60,8 @@ public void setup() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
FileSystemTimelineWriterImpl.class, TimelineWriter.class);
conf.set(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES,
"30000-30100");
collectorManager.init(conf);
collectorManager.start();
}
@ -83,7 +85,8 @@ public void testStartWebApp() throws Exception {
String[] parts = address.split(":");
assertEquals(2, parts.length);
assertNotNull(parts[0]);
assertTrue(Integer.valueOf(parts[1]) > 0);
assertTrue(Integer.valueOf(parts[1]) >= 30000 &&
Integer.valueOf(parts[1]) <= 30100);
}
@Test(timeout=60000)
@ -153,7 +156,6 @@ public Boolean call() {
private NodeTimelineCollectorManager createCollectorManager() {
final NodeTimelineCollectorManager cm =
spy(new NodeTimelineCollectorManager());
doReturn(new Configuration()).when(cm).getConfig();
CollectorNodemanagerProtocol nmCollectorService =
mock(CollectorNodemanagerProtocol.class);
GetTimelineCollectorContextResponse response =