big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8934. [BackPort][GPG] Add JvmMetricsInfo and pause monitor. (#5929) Contributed by Bilwa S T.

Browse Source
This commit is contained in:
slfan1989 2023-08-12 09:15:34 +08:00 committed by GitHub
parent ad2f45c64f
commit 8d95c588d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 406 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -4445,6 +4445,27 @@ public static boolean isAclEnabled(Configuration conf) {
FEDERATION_GPG_LOAD_BASED_PREFIX + "scaling";
public static final String DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING = "LINEAR";
public static final String GPG_WEBAPP_PREFIX = FEDERATION_GPG_PREFIX + "webapp.";
/** Enable/disable CORS filter. */
public static final String GPG_WEBAPP_ENABLE_CORS_FILTER =
GPG_WEBAPP_PREFIX + "cross-origin.enabled";
public static final boolean DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER = false;
/** The address of the GPG web application. */
public static final String GPG_WEBAPP_ADDRESS = GPG_WEBAPP_PREFIX + "address";
public static final int DEFAULT_GPG_WEBAPP_PORT = 8069;
public static final String DEFAULT_GPG_WEBAPP_ADDRESS =
"0.0.0.0:" + DEFAULT_GPG_WEBAPP_PORT;
/** The https address of the GPG web application. */
public static final String GPG_WEBAPP_HTTPS_ADDRESS = GPG_WEBAPP_PREFIX + "https.address";
public static final int DEFAULT_GPG_WEBAPP_HTTPS_PORT = 8070;
public static final String DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS =
"0.0.0.0:" + DEFAULT_GPG_WEBAPP_HTTPS_PORT;
/**
* Connection and Read timeout from the Router to RM.
*/

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
View File

@ -271,6 +271,7 @@
<exclude>src/main/resources/webapps/test/.keep</exclude>
<exclude>src/main/resources/webapps/proxy/.keep</exclude>
<exclude>src/main/resources/webapps/node/.keep</exclude>
<exclude>src/main/resources/webapps/gpg/.keep</exclude>
<exclude>src/main/resources/webapps/static/dt-1.11.5/css/jquery.dataTables.css</exclude>
<exclude>src/main/resources/webapps/static/dt-1.11.5/css/custom_datatable.css</exclude>
<exclude>src/main/resources/webapps/static/dt-1.11.5/css/jui-dt.css</exclude>

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
View File

@ -183,6 +183,16 @@ public static String getRouterWebAppURLWithoutScheme(Configuration conf) {
}
}
public static String getGPGWebAppURLWithoutScheme(Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
return conf.get(YarnConfiguration.GPG_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS);
} else {
return conf.get(YarnConfiguration.GPG_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_GPG_WEBAPP_ADDRESS);
}
}
public static List<String> getProxyHostsAndPortsForAmFilter(
Configuration conf) {
List<String> addrs = new ArrayList<String>();

0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/gpg/.keep Normal file
View File

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -5530,4 +5530,32 @@
<value>LINEAR</value>
</property>
<property>
<description>Flag to enable cross-origin (CORS) support in the GPG. This flag
requires the CORS filter initializer to be added to the filter initializers
list in core-site.xml.</description>
<name>yarn.federation.gpg.webapp.cross-origin.enabled</name>
<value>false</value>
</property>
<property>
<description>
The http address of the GPG web application.
If only a host is provided as the value,
the webapp will be served on a random port.
</description>
<name>yarn.federation.gpg.webapp.address</name>
<value>0.0.0.0:8069</value>
</property>
<property>
<description>
The https address of the GPG web application.
If only a host is provided as the value,
the webapp will be served on a random port.
</description>
<name> yarn.federation.gpg.webapp.https.address</name>
<value>0.0.0.0:8070</value>
</property>
</configuration>

70
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
View File

@ -21,15 +21,22 @@
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.time.DurationFormatUtils;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.JvmPauseMonitor;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
@ -38,6 +45,10 @@
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.PolicyGenerator;
import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner;
import org.apache.hadoop.yarn.server.globalpolicygenerator.webapp.GPGWebApp;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -61,6 +72,7 @@ public class GlobalPolicyGenerator extends CompositeService {
public static final int SHUTDOWN_HOOK_PRIORITY = 30;
private AtomicBoolean isStopping = new AtomicBoolean(false);
private static final String METRICS_NAME = "Global Policy Generator";
private static long gpgStartupTime = System.currentTimeMillis();
// Federation Variables
private GPGContext gpgContext;
@ -69,6 +81,9 @@ public class GlobalPolicyGenerator extends CompositeService {
private ScheduledThreadPoolExecutor scheduledExecutorService;
private SubClusterCleaner subClusterCleaner;
private PolicyGenerator policyGenerator;
private String webAppAddress;
private JvmPauseMonitor pauseMonitor;
private WebApp webApp;
public GlobalPolicyGenerator() {
super(GlobalPolicyGenerator.class.getName());
@ -107,7 +122,12 @@ protected void serviceInit(Configuration conf) throws Exception {
this.subClusterCleaner = new SubClusterCleaner(conf, this.gpgContext);
this.policyGenerator = new PolicyGenerator(conf, this.gpgContext);
this.webAppAddress = WebAppUtils.getGPGWebAppURLWithoutScheme(conf);
DefaultMetricsSystem.initialize(METRICS_NAME);
JvmMetrics jm = JvmMetrics.initSingleton("GPG", null);
pauseMonitor = new JvmPauseMonitor();
addService(pauseMonitor);
jm.setPauseMonitor(pauseMonitor);
// super.serviceInit after all services are added
super.serviceInit(conf);
@ -163,6 +183,7 @@ protected void serviceStart() throws Exception {
LOG.info("Scheduled policy-generator with interval: {}",
DurationFormatUtils.formatDurationISO(policyGeneratorIntervalMillis));
}
startWepApp();
}
@Override
@ -181,6 +202,9 @@ protected void serviceStop() throws Exception {
if (this.isStopping.getAndSet(true)) {
return;
}
if (webApp != null) {
webApp.stop();
}
DefaultMetricsSystem.shutdown();
super.serviceStop();
}
@ -193,6 +217,43 @@ public GPGContext getGPGContext() {
return this.gpgContext;
}
@VisibleForTesting
public void startWepApp() {
Configuration configuration = getConfig();
boolean enableCors = configuration.getBoolean(YarnConfiguration.GPG_WEBAPP_ENABLE_CORS_FILTER,
YarnConfiguration.DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER);
if (enableCors) {
configuration.setBoolean(HttpCrossOriginFilterInitializer.PREFIX
+ HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
}
// Always load pseudo authentication filter to parse "user.name" in an URL
// to identify a HTTP request's user.
boolean hasHadoopAuthFilterInitializer = false;
String filterInitializerConfKey = "hadoop.http.filter.initializers";
Class<?>[] initializersClasses = configuration.getClasses(filterInitializerConfKey);
List<String> targets = new ArrayList<>();
if (initializersClasses != null) {
for (Class<?> initializer : initializersClasses) {
if (initializer.getName().equals(AuthenticationFilterInitializer.class.getName())) {
hasHadoopAuthFilterInitializer = true;
break;
}
targets.add(initializer.getName());
}
}
if (!hasHadoopAuthFilterInitializer) {
targets.add(AuthenticationFilterInitializer.class.getName());
configuration.set(filterInitializerConfKey, StringUtils.join(",", targets));
}
LOG.info("Instantiating GPGWebApp at {}.", webAppAddress);
GPGWebApp gpgWebApp = new GPGWebApp(this);
webApp = WebApps.$for("gpg").at(webAppAddress).start(gpgWebApp);
}
@SuppressWarnings("resource")
public static void startGPG(String[] argv, Configuration conf) {
boolean federationEnabled = conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED,
@ -232,4 +293,13 @@ public static void main(String[] argv) {
System.exit(-1);
}
}
public static long getGPGStartupTime() {
return gpgStartupTime;
}
@VisibleForTesting
public WebApp getWebApp() {
return webApp;
}
}

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import org.apache.hadoop.yarn.webapp.Controller;
import com.google.inject.Inject;
/**
* Controller for the GPG Web UI.
*/
public class GPGController extends Controller {
@Inject
GPGController(RequestContext ctx) {
super(ctx);
}
@Override
public void index() {
setTitle("GPG");
render(GPGOverviewPage.class);
}
public void overview() {
setTitle("GPG Details");
render(GPGOverviewPage.class);
}
}

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewBlock.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import java.util.Date;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
/**
* Overview block for the GPG Web UI.
*/
public class GPGOverviewBlock extends HtmlBlock {
@Inject
GPGOverviewBlock(GlobalPolicyGenerator gpg, ViewContext ctx) {
super(ctx);
}
@Override
protected void render(Block html) {
info("GPG Details")
.__("GPG started on", new Date(GlobalPolicyGenerator.getGPGStartupTime()))
.__("GPG Version", YarnVersionInfo.getVersion())
.__("Hadoop Version", VersionInfo.getVersion());
html.__(InfoBlock.class);
}
}

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewPage.java Normal file
View File

@ -0,0 +1,52 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
/**
* Overview page for the GPG Web UI.
*/
public class GPGOverviewPage extends TwoColumnLayout {
@Override
protected void preHead(Page.HTML<__> html) {
commonPreHead(html);
setTitle("GPG");
}
protected void commonPreHead(Page.HTML<__> html) {
set(ACCORDION_ID, "nav");
set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
}
@Override
protected Class<? extends SubView> nav() {
return NavBlock.class;
}
@Override
protected Class<? extends SubView> content() {
return GPGOverviewBlock.class;
}
}

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java Normal file
View File

@ -0,0 +1,45 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
/**
* The GPG webapp.
*/
public class GPGWebApp extends WebApp{
private GlobalPolicyGenerator gpg;
public GPGWebApp(GlobalPolicyGenerator gpg) {
this.gpg = gpg;
}
@Override
public void setup() {
bind(JAXBContextResolver.class);
bind(GPGWebApp.class).toInstance(this);
bind(GenericExceptionHandler.class);
if (gpg != null) {
bind(GlobalPolicyGenerator.class).toInstance(gpg);
}
route("/", GPGController.class, "overview");
}
}

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java Normal file
View File

@ -0,0 +1,42 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
/**
* Navigation block for the GPG Web UI.
*/
public class NavBlock extends HtmlBlock {
@Override
public void render(Block html) {
html.
div("#nav").
h3("GPG").
ul().
li().a(url(""), "Overview").__().
__().
h3("Tools").
ul().
li().a("/conf", "Configuration").__().
li().a("/logs", "Local logs").__().
li().a("/stacks", "Server stacks").__().
li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__();
}
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/package-info.java Normal file
View File

@ -0,0 +1,24 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Classes comprising the policy generator for the GPG. Responsibilities include
* generating and updating policies based on the cluster status.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java
View File

@ -19,9 +19,14 @@
package org.apache.hadoop.yarn.server.globalpolicygenerator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Test;
import java.util.List;
import java.util.concurrent.TimeoutException;
/**
* Unit test for GlobalPolicyGenerator.
*/
@ -35,4 +40,19 @@ public void testNonFederation() {
// If GPG starts running, this call will not return
GlobalPolicyGenerator.startGPG(new String[0], conf);
}
@Test
public void testGpgWithFederation() throws InterruptedException, TimeoutException {
// In this test case, we hope that gpg can start normally in federation mode.
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
GlobalPolicyGenerator gpg = new GlobalPolicyGenerator();
gpg.initAndStart(conf, false);
GenericTestUtils.waitFor(() -> {
List<Service> services = gpg.getServices();
return (services.size() == 1 && gpg.getWebApp() != null);
}, 100, 5000);
}
}