big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3374. Collector's web server should randomly bind an available port. Contributed by Zhijie Shen

Browse Source 
(cherry picked from commit 3aa898e734a1e4368ddf1d0bbd31f9b4de53ceba)
This commit is contained in:
Junping Du 2015-04-02 11:59:59 -07:00 committed by Sangjin Lee
parent d67c9bdb4d
commit 42e49399ce
3 changed files with 23 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -2012,6 +2012,7 @@ public static boolean isAclEnabled(Configuration conf) {
/** The listening endpoint for the timeline service application.*/
public static final String TIMELINE_SERVICE_BIND_HOST =
TIMELINE_SERVICE_PREFIX + "bind-host";
public static final String DEFAULT_TIMELINE_SERVICE_BIND_HOST = "0.0.0.0";
/** The number of threads to handle client RPC API requests. */
public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT =

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
View File

@ -210,22 +210,17 @@ public boolean containsKey(String id) {
*/
private void startWebApp() {
Configuration conf = getConfig();
// use the same ports as the old ATS for now; we could create new properties
// for the new timeline service if needed
String bindAddress = WebAppUtils.getWebAppBindURL(conf,
YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
NetUtils.createSocketAddr(bindAddress));
LOG.info("Instantiating the per-node collector webapp at " +
timelineRestServerBindAddress);
String bindAddress = conf.get(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
try {
Configuration confForInfoServer = new Configuration(conf);
confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
HttpServer2.Builder builder = new HttpServer2.Builder()
.setName("timeline")
.setConf(conf)
.addEndpoint(URI.create("http://" + bindAddress));
.addEndpoint(URI.create(
(YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
bindAddress));
timelineRestServer = builder.build();
// TODO: replace this by an authentication filter in future.
HashMap<String, String> options = new HashMap<>();
@ -249,6 +244,11 @@ private void startWebApp() {
LOG.error(msg, e);
throw new YarnRuntimeException(msg, e);
}
//TODO: We need to think of the case of multiple interfaces
this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
timelineRestServer.getConnectorAddress(0));
LOG.info("Instantiated the per-node collector webapp at " +
timelineRestServerBindAddress);
}
private void reportNewCollectorToNM(ApplicationId appId)

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
View File

@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.any;
@ -63,6 +65,16 @@ public void tearDown() throws Exception {
}
}
@Test
public void testStartWebApp() throws Exception {
assertNotNull(collectorManager.getRestServerBindAddress());
String address = collectorManager.getRestServerBindAddress();
String[] parts = address.split(":");
assertEquals(2, parts.length);
assertNotNull(parts[0]);
assertTrue(Integer.valueOf(parts[1]) > 0);
}
@Test(timeout=60000)
public void testMultithreadedAdd() throws Exception {
final int NUM_APPS = 5;