big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles)

Browse Source
This commit is contained in:
Jonathan Eagles 2015-10-21 15:38:08 -05:00
parent d759b4bd74
commit 25f8f801d1
7 changed files with 140 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -989,6 +989,8 @@ Release 2.7.2 - UNRELEASED
YARN-2801. Add documentation for node labels feature. (Wangda Tan and Naganarasimha
G R via ozawa)
YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles)
OPTIMIZATIONS
BUG FIXES

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1524,6 +1524,23 @@ private static void addDeprecatedKeys() {
public static final String TIMELINE_SERVICE_PREFIX =
YARN_PREFIX + "timeline-service.";
/**
* Comma seperated list of names for UIs hosted in the timeline server
* (For pluggable UIs).
*/
public static final String TIMELINE_SERVICE_UI_NAMES =
TIMELINE_SERVICE_PREFIX + "ui-names";
/** Relative web path that will serve up this UI (For pluggable UIs). */
public static final String TIMELINE_SERVICE_UI_WEB_PATH_PREFIX =
TIMELINE_SERVICE_PREFIX + "ui-web-path.";
/**
* Path to war file or static content directory for this UI
* (For pluggable UIs).
*/
public static final String TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX =
TIMELINE_SERVICE_PREFIX + "ui-on-disk-path.";
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
View File

@ -166,7 +166,7 @@ public Builder<T> inDevMode() {
return this;
}
public WebApp start(WebApp webapp) {
public WebApp build(WebApp webapp) {
if (webapp == null) {
webapp = new WebApp() {
@Override
@ -271,8 +271,7 @@ public void setup() {
webapp.setConf(conf);
webapp.setHttpServer(server);
server.start();
LOG.info("Web app /"+ name +" started at "+ server.getConnectorAddress(0).getPort());
} catch (ClassNotFoundException e) {
throw new WebAppException("Error starting http server", e);
} catch (IOException e) {
@ -300,6 +299,19 @@ public WebApp start() {
return start(null);
}
public WebApp start(WebApp webapp) {
WebApp webApp = build(webapp);
HttpServer2 httpServer = webApp.httpServer();
try {
httpServer.start();
LOG.info("Web app " + name + " started at "
+ httpServer.getConnectorAddress(0).getPort());
} catch (IOException e) {
throw new WebAppException("Error starting http server", e);
}
return webApp;
}
private String inferHostClass() {
String thisClass = this.getClass().getName();
Throwable t = new Throwable();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1868,6 +1868,12 @@
<value>/etc/krb5.keytab</value>
</property>
<property>
<description>Comma separated list of UIs that will be hosted</description>
<name>yarn.timeline-service.ui-names</name>
<value></value>
</property>
<property>
<description>
Default maximum number of retries for timeline service client

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
View File

@ -26,6 +26,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
@ -54,6 +55,8 @@
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.mortbay.jetty.servlet.FilterHolder;
import org.mortbay.jetty.webapp.WebAppContext;
import com.google.common.annotations.VisibleForTesting;
@ -137,6 +140,12 @@ ApplicationHistoryClientService getClientService() {
return this.ahsClientService;
}
@Private
@VisibleForTesting
int getPort() {
return this.webApp.httpServer().getConnectorAddress(0).getPort();
}
/**
* @return ApplicationTimelineStore
*/
@ -223,6 +232,7 @@ private TimelineDataManager createTimelineDataManager(Configuration conf) {
timelineStore, new TimelineACLsManager(conf));
}
@SuppressWarnings("unchecked")
private void startWebApp() {
Configuration conf = getConfig();
TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager(
@ -278,14 +288,42 @@ private void startWebApp() {
String bindAddress = WebAppUtils.getWebAppBindURL(conf,
YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
LOG.info("Instantiating AHSWebApp at " + bindAddress);
try {
AHSWebApp ahsWebApp = new AHSWebApp(timelineDataManager, ahsClientService);
webApp =
WebApps
.$for("applicationhistory", ApplicationHistoryClientService.class,
ahsClientService, "ws")
.with(conf).at(bindAddress).start(
new AHSWebApp(timelineDataManager, ahsClientService));
.with(conf).withAttribute(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS)).at(bindAddress).build(ahsWebApp);
HttpServer2 httpServer = webApp.httpServer();
String[] names = conf.getTrimmedStrings(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES);
WebAppContext webAppContext = httpServer.getWebAppContext();
for (String name : names) {
String webPath = conf.get(
YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + name);
String onDiskPath = conf.get(
YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX + name);
WebAppContext uiWebAppContext = new WebAppContext();
uiWebAppContext.setContextPath(webPath);
uiWebAppContext.setWar(onDiskPath);
final String[] ALL_URLS = { "/*" };
FilterHolder[] filterHolders =
webAppContext.getServletHandler().getFilters();
for (FilterHolder filterHolder: filterHolders) {
if (!"guice".equals(filterHolder.getName())) {
HttpServer2.defineFilter(uiWebAppContext, filterHolder.getName(),
filterHolder.getClassName(), filterHolder.getInitParameters(),
ALL_URLS);
}
}
LOG.info("Hosting " + name + " from " + onDiskPath + " at " + webPath);
httpServer.addContext(uiWebAppContext, true);
}
httpServer.start();
LOG.info("Instantiating AHSWebApp at " + getPort());
} catch (Exception e) {
String msg = "AHSWebApp failed to start.";
LOG.error(msg, e);

53
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
View File

@ -28,16 +28,20 @@
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
@ -173,4 +177,49 @@ public void testFilterOverrides() throws Exception {
}
}
@Test(timeout = 240000)
public void testHostedUIs() throws Exception {
ApplicationHistoryServer historyServer = new ApplicationHistoryServer();
Configuration config = new YarnConfiguration();
config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
MemoryTimelineStore.class, TimelineStore.class);
config.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
MemoryTimelineStateStore.class, TimelineStateStore.class);
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
"localhost:0");
final String UI1 = "UI1";
String connFileStr = "";
File diskFile = new File("./pom.xml");
String diskFileStr = readInputStream(new FileInputStream(diskFile));
try {
config.set(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES, UI1);
config.set(YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + UI1,
"/" + UI1);
config.set(YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX
+ UI1, "./");
historyServer.init(config);
historyServer.start();
URL url = new URL("http://localhost:" + historyServer.getPort() + "/"
+ UI1 + "/pom.xml");
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.connect();
assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
connFileStr = readInputStream(conn.getInputStream());
} finally {
historyServer.stop();
}
assertEquals("Web file contents should be the same as on disk contents",
diskFileStr, connFileStr);
}
private String readInputStream(InputStream input) throws Exception {
ByteArrayOutputStream data = new ByteArrayOutputStream();
byte[] buffer = new byte[512];
int read;
while ((read = input.read(buffer)) >= 0) {
data.write(buffer, 0, read);
}
return new String(data.toByteArray(), "UTF-8");
}
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
View File

@ -185,6 +185,14 @@ selected if this policy is either of `HTTPS_ONLY` or `HTTP_AND_HTTPS`.
| `yarn.timeline-service.client.retry-interval-ms` | The interval in milliseconds between retries for the timeline service client. Defaults to `1000`. |
| `yarn.timeline-service.generic-application-history.max-applications` | The max number of applications could be fetched by using REST API or application history protocol and shown in timeline server web ui. Defaults to `10000`. |
#### UI Hosting Configuration
The timeline service can host multiple UIs if enabled. The service can support both static web sites hosted in a directory or war files bundled. The web UI is then hosted on the timeline service HTTP port under the path configured.
| Configuration Property | Description |
|:---- |:---- |
| `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be hosted. Defaults to `none`. |
| `yarn.timeline-service.ui-on-disk-path.$name` | For each of the ui-names, an on disk path should be specified to the directory service static content or the location of a web archive (war file). |
| `yarn.timeline-service.ui-web-path.$name` | For each of the ui-names, the web path should be specified relative to the Timeline server root. Paths should begin with a starting slash. |
#### Security Configuration