big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2203. [YARN-1492] Web UI for cache manager. (Chris Trezzo via kasha)

Browse Source
This commit is contained in:
Karthik Kambatla 2014-12-17 16:32:21 -08:00
parent a1bd140964
commit b7f64823e1
8 changed files with 322 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -65,6 +65,8 @@ Release 2.7.0 - UNRELEASED
YARN-2880. Added a test to make sure node labels will be recovered
if RM restart is enabled. (Rohith Sharmaks via jianhe)
YARN-2203. [YARN-1492] Web UI for cache manager. (Chris Trezzo via kasha)
IMPROVEMENTS
YARN-2950. Change message to mandate, not suggest JS requirement on UI.

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1413,6 +1413,13 @@ private static void addDeprecatedKeys() {
SHARED_CACHE_PREFIX + "admin.thread-count";
public static final int DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT = 1;
/** The address of the SCM web application. */
public static final String SCM_WEBAPP_ADDRESS =
SHARED_CACHE_PREFIX + "webapp.address";
public static final int DEFAULT_SCM_WEBAPP_PORT = 8788;
public static final String DEFAULT_SCM_WEBAPP_ADDRESS =
"0.0.0.0:" + DEFAULT_SCM_WEBAPP_PORT;
// In-memory SCM store configuration
public static final String IN_MEMORY_STORE_PREFIX =

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1444,6 +1444,12 @@
<value>1</value>
</property>
<property>
<description>The address of the web application in the SCM (shared cache manager)</description>
<name>yarn.sharedcache.webapp.address</name>
<value>0.0.0.0:8788</value>
</property>
<property>
<description>The frequency at which a cleaner task runs.
Specified in minutes.</description>

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java
View File

@ -33,6 +33,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore;
import org.apache.hadoop.yarn.server.sharedcachemanager.webapp.SCMWebServer;
import com.google.common.annotations.VisibleForTesting;
@ -77,6 +78,9 @@ protected void serviceInit(Configuration conf) throws Exception {
SCMAdminProtocolService saps = createSCMAdminProtocolService(cs);
addService(saps);
SCMWebServer webUI = createSCMWebServer(this);
addService(webUI);
// init metrics
DefaultMetricsSystem.initialize("SharedCacheManager");
JvmMetrics.initSingleton("SharedCacheManager", null);
@ -121,6 +125,10 @@ private SCMAdminProtocolService createSCMAdminProtocolService(
return new SCMAdminProtocolService(cleanerService);
}
private SCMWebServer createSCMWebServer(SharedCacheManager scm) {
return new SCMWebServer(scm);
}
@Override
protected void serviceStop() throws Exception {

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java Normal file
View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.webapp.Controller;
/**
* The controller class for the shared cache manager web app.
*/
@Private
@Unstable
public class SCMController extends Controller {
@Override
public void index() {
setTitle("Shared Cache Manager");
}
/**
* It is referenced in SCMWebServer.SCMWebApp.setup()
*/
@SuppressWarnings("unused")
public void overview() {
render(SCMOverviewPage.class);
}
}

70
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java Normal file
View File

@ -0,0 +1,70 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics;
import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics;
import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.SharedCacheUploaderMetrics;
/**
* This class is used to summarize useful shared cache manager metrics for the
* webUI display.
*/
@XmlRootElement(name = "SCMMetrics")
@XmlAccessorType(XmlAccessType.FIELD)
@Private
@Unstable
public class SCMMetricsInfo {
protected long totalDeletedFiles;
protected long totalProcessedFiles;
protected long cacheHits;
protected long cacheMisses;
protected long cacheReleases;
protected long acceptedUploads;
protected long rejectedUploads;
public SCMMetricsInfo() {
}
public SCMMetricsInfo(CleanerMetrics cleanerMetrics,
ClientSCMMetrics clientSCMMetrics,
SharedCacheUploaderMetrics scmUploaderMetrics) {
totalDeletedFiles = cleanerMetrics.getTotalDeletedFiles();
totalProcessedFiles = cleanerMetrics.getTotalProcessedFiles();
cacheHits = clientSCMMetrics.getCacheHits();
cacheMisses = clientSCMMetrics.getCacheMisses();
cacheReleases = clientSCMMetrics.getCacheReleases();
acceptedUploads = scmUploaderMetrics.getAcceptedUploads();
rejectedUploads = scmUploaderMetrics.getRejectUploads();
}
public long getTotalDeletedFiles() { return totalDeletedFiles; }
public long getTotalProcessedFiles() { return totalProcessedFiles; }
public long getCacheHits() { return cacheHits; }
public long getCacheMisses() { return cacheMisses; }
public long getCacheReleases() { return cacheReleases; }
public long getAcceptedUploads() { return acceptedUploads; }
public long getRejectUploads() { return rejectedUploads; }
}

95
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java Normal file
View File

@ -0,0 +1,95 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager;
import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics;
import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics;
import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.SharedCacheUploaderMetrics;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
import com.google.inject.Inject;
/**
* This class is to render the shared cache manager web ui overview page.
*/
@Private
@Unstable
public class SCMOverviewPage extends TwoColumnLayout {
@Override protected void preHead(Page.HTML<_> html) {
set(ACCORDION_ID, "nav");
set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
}
@Override protected Class<? extends SubView> content() {
return SCMOverviewBlock.class;
}
@Override
protected Class<? extends SubView> nav() {
return SCMOverviewNavBlock.class;
}
static private class SCMOverviewNavBlock extends HtmlBlock {
@Override
protected void render(Block html) {
html.div("#nav").h3("Tools").ul().li().a("/conf", "Configuration")._()
.li().a("/stacks", "Thread dump")._().li().a("/logs", "Logs")._()
.li().a("/metrics", "Metrics")._()._()._();
}
}
static private class SCMOverviewBlock extends HtmlBlock {
final SharedCacheManager scm;
@Inject
SCMOverviewBlock(SharedCacheManager scm, ViewContext ctx) {
super(ctx);
this.scm = scm;
}
@Override
protected void render(Block html) {
SCMMetricsInfo metricsInfo = new SCMMetricsInfo(
CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(),
SharedCacheUploaderMetrics.getInstance());
info("Shared Cache Manager overview").
_("Started on:", Times.format(scm.getStartTime())).
_("Cache hits: ", metricsInfo.getCacheHits()).
_("Cache misses: ", metricsInfo.getCacheMisses()).
_("Cache releases: ", metricsInfo.getCacheReleases()).
_("Accepted uploads: ", metricsInfo.getAcceptedUploads()).
_("Rejected uploads: ", metricsInfo.getRejectUploads()).
_("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()).
_("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles());
html._(InfoBlock.class);
}
}
}

91
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java Normal file
View File

@ -0,0 +1,91 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
/**
* A very simple web interface for the metrics reported by
* {@link org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager}
* TODO: Security for web ui (See YARN-2774)
*/
@Private
@Unstable
public class SCMWebServer extends AbstractService {
private static final Log LOG = LogFactory.getLog(SCMWebServer.class);
private final SharedCacheManager scm;
private WebApp webApp;
private String bindAddress;
public SCMWebServer(SharedCacheManager scm) {
super(SCMWebServer.class.getName());
this.scm = scm;
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
this.bindAddress = getBindAddress(conf);
super.serviceInit(conf);
}
private String getBindAddress(Configuration conf) {
return conf.get(YarnConfiguration.SCM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_SCM_WEBAPP_ADDRESS);
}
@Override
protected void serviceStart() throws Exception {
SCMWebApp scmWebApp = new SCMWebApp(scm);
this.webApp = WebApps.$for("sharedcache").at(bindAddress).start(scmWebApp);
LOG.info("Instantiated " + SCMWebApp.class.getName() + " at " + bindAddress);
}
@Override
protected void serviceStop() throws Exception {
if (this.webApp != null) {
this.webApp.stop();
}
}
private class SCMWebApp extends WebApp {
private final SharedCacheManager scm;
public SCMWebApp(SharedCacheManager scm) {
this.scm = scm;
}
@Override
public void setup() {
if (scm != null) {
bind(SharedCacheManager.class).toInstance(scm);
}
route("/", SCMController.class, "overview");
}
}
}