diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4c5fc77996..7af08b92d2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -65,6 +65,8 @@ Release 2.7.0 - UNRELEASED
YARN-2880. Added a test to make sure node labels will be recovered
if RM restart is enabled. (Rohith Sharmaks via jianhe)
+ YARN-2203. [YARN-1492] Web UI for cache manager. (Chris Trezzo via kasha)
+
IMPROVEMENTS
YARN-2950. Change message to mandate, not suggest JS requirement on UI.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 55073c53e0..d0cf761572 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1413,6 +1413,13 @@ private static void addDeprecatedKeys() {
SHARED_CACHE_PREFIX + "admin.thread-count";
public static final int DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT = 1;
+ /** The address of the SCM web application. */
+ public static final String SCM_WEBAPP_ADDRESS =
+ SHARED_CACHE_PREFIX + "webapp.address";
+ public static final int DEFAULT_SCM_WEBAPP_PORT = 8788;
+ public static final String DEFAULT_SCM_WEBAPP_ADDRESS =
+ "0.0.0.0:" + DEFAULT_SCM_WEBAPP_PORT;
+
// In-memory SCM store configuration
public static final String IN_MEMORY_STORE_PREFIX =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 73a6b5d55e..ff4bf2bc44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1444,6 +1444,12 @@
1
+
+ The address of the web application in the SCM (shared cache manager)
+ yarn.sharedcache.webapp.address
+ 0.0.0.0:8788
+
+
The frequency at which a cleaner task runs.
Specified in minutes.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java
index d22fa5118b..331e29ee32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java
@@ -33,6 +33,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore;
+import org.apache.hadoop.yarn.server.sharedcachemanager.webapp.SCMWebServer;
import com.google.common.annotations.VisibleForTesting;
@@ -77,6 +78,9 @@ protected void serviceInit(Configuration conf) throws Exception {
SCMAdminProtocolService saps = createSCMAdminProtocolService(cs);
addService(saps);
+ SCMWebServer webUI = createSCMWebServer(this);
+ addService(webUI);
+
// init metrics
DefaultMetricsSystem.initialize("SharedCacheManager");
JvmMetrics.initSingleton("SharedCacheManager", null);
@@ -121,6 +125,10 @@ private SCMAdminProtocolService createSCMAdminProtocolService(
return new SCMAdminProtocolService(cleanerService);
}
+ private SCMWebServer createSCMWebServer(SharedCacheManager scm) {
+ return new SCMWebServer(scm);
+ }
+
@Override
protected void serviceStop() throws Exception {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java
new file mode 100644
index 0000000000..f597d14bf4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.webapp.Controller;
+
+/**
+ * The controller class for the shared cache manager web app.
+ */
+@Private
+@Unstable
+public class SCMController extends Controller {
+ @Override
+ public void index() {
+ setTitle("Shared Cache Manager");
+ }
+
+ /**
+ * It is referenced in SCMWebServer.SCMWebApp.setup()
+ */
+ @SuppressWarnings("unused")
+ public void overview() {
+ render(SCMOverviewPage.class);
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java
new file mode 100644
index 0000000000..24aa1452a6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.SharedCacheUploaderMetrics;
+
+/**
+ * This class is used to summarize useful shared cache manager metrics for the
+ * webUI display.
+ */
+@XmlRootElement(name = "SCMMetrics")
+@XmlAccessorType(XmlAccessType.FIELD)
+@Private
+@Unstable
+public class SCMMetricsInfo {
+ protected long totalDeletedFiles;
+ protected long totalProcessedFiles;
+ protected long cacheHits;
+ protected long cacheMisses;
+ protected long cacheReleases;
+ protected long acceptedUploads;
+ protected long rejectedUploads;
+
+ public SCMMetricsInfo() {
+ }
+
+ public SCMMetricsInfo(CleanerMetrics cleanerMetrics,
+ ClientSCMMetrics clientSCMMetrics,
+ SharedCacheUploaderMetrics scmUploaderMetrics) {
+ totalDeletedFiles = cleanerMetrics.getTotalDeletedFiles();
+ totalProcessedFiles = cleanerMetrics.getTotalProcessedFiles();
+ cacheHits = clientSCMMetrics.getCacheHits();
+ cacheMisses = clientSCMMetrics.getCacheMisses();
+ cacheReleases = clientSCMMetrics.getCacheReleases();
+ acceptedUploads = scmUploaderMetrics.getAcceptedUploads();
+ rejectedUploads = scmUploaderMetrics.getRejectUploads();
+ }
+
+ public long getTotalDeletedFiles() { return totalDeletedFiles; }
+ public long getTotalProcessedFiles() { return totalProcessedFiles; }
+ public long getCacheHits() { return cacheHits; }
+ public long getCacheMisses() { return cacheMisses; }
+ public long getCacheReleases() { return cacheReleases; }
+ public long getAcceptedUploads() { return acceptedUploads; }
+ public long getRejectUploads() { return rejectedUploads; }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java
new file mode 100644
index 0000000000..27944d3919
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java
@@ -0,0 +1,95 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.SharedCacheUploaderMetrics;
+import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
+
+import com.google.inject.Inject;
+
+/**
+ * This class is to render the shared cache manager web ui overview page.
+ */
+@Private
+@Unstable
+public class SCMOverviewPage extends TwoColumnLayout {
+
+ @Override protected void preHead(Page.HTML<_> html) {
+ set(ACCORDION_ID, "nav");
+ set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
+ }
+
+ @Override protected Class extends SubView> content() {
+ return SCMOverviewBlock.class;
+ }
+
+ @Override
+ protected Class extends SubView> nav() {
+ return SCMOverviewNavBlock.class;
+ }
+
+ static private class SCMOverviewNavBlock extends HtmlBlock {
+ @Override
+ protected void render(Block html) {
+ html.div("#nav").h3("Tools").ul().li().a("/conf", "Configuration")._()
+ .li().a("/stacks", "Thread dump")._().li().a("/logs", "Logs")._()
+ .li().a("/metrics", "Metrics")._()._()._();
+ }
+ }
+
+ static private class SCMOverviewBlock extends HtmlBlock {
+ final SharedCacheManager scm;
+
+ @Inject
+ SCMOverviewBlock(SharedCacheManager scm, ViewContext ctx) {
+ super(ctx);
+ this.scm = scm;
+ }
+
+ @Override
+ protected void render(Block html) {
+ SCMMetricsInfo metricsInfo = new SCMMetricsInfo(
+ CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(),
+ SharedCacheUploaderMetrics.getInstance());
+ info("Shared Cache Manager overview").
+ _("Started on:", Times.format(scm.getStartTime())).
+ _("Cache hits: ", metricsInfo.getCacheHits()).
+ _("Cache misses: ", metricsInfo.getCacheMisses()).
+ _("Cache releases: ", metricsInfo.getCacheReleases()).
+ _("Accepted uploads: ", metricsInfo.getAcceptedUploads()).
+ _("Rejected uploads: ", metricsInfo.getRejectUploads()).
+ _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()).
+ _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles());
+ html._(InfoBlock.class);
+ }
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java
new file mode 100644
index 0000000000..b81ed29f1c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.WebApps;
+
+/**
+ * A very simple web interface for the metrics reported by
+ * {@link org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager}
+ * TODO: Security for web ui (See YARN-2774)
+ */
+@Private
+@Unstable
+public class SCMWebServer extends AbstractService {
+ private static final Log LOG = LogFactory.getLog(SCMWebServer.class);
+
+ private final SharedCacheManager scm;
+ private WebApp webApp;
+ private String bindAddress;
+
+ public SCMWebServer(SharedCacheManager scm) {
+ super(SCMWebServer.class.getName());
+ this.scm = scm;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ this.bindAddress = getBindAddress(conf);
+ super.serviceInit(conf);
+ }
+
+ private String getBindAddress(Configuration conf) {
+ return conf.get(YarnConfiguration.SCM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_SCM_WEBAPP_ADDRESS);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ SCMWebApp scmWebApp = new SCMWebApp(scm);
+ this.webApp = WebApps.$for("sharedcache").at(bindAddress).start(scmWebApp);
+ LOG.info("Instantiated " + SCMWebApp.class.getName() + " at " + bindAddress);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (this.webApp != null) {
+ this.webApp.stop();
+ }
+ }
+
+ private class SCMWebApp extends WebApp {
+ private final SharedCacheManager scm;
+
+ public SCMWebApp(SharedCacheManager scm) {
+ this.scm = scm;
+ }
+
+ @Override
+ public void setup() {
+ if (scm != null) {
+ bind(SharedCacheManager.class).toInstance(scm);
+ }
+ route("/", SCMController.class, "overview");
+ }
+ }
+}
\ No newline at end of file