big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10439. Yarn Service AM listens on all IP's on the machine. Contributed by D M Murali Krishna Reddy

Browse Source 
(cherry picked from commit d0dcfc405c)
This commit is contained in:
Brahma Reddy Battula 2021-03-30 09:46:12 +05:30
parent e9d8f16a70
commit 5181b2004b
5 changed files with 33 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.Server;
@ -53,8 +54,10 @@
import org.apache.hadoop.yarn.service.component.ComponentEvent;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.apache.hadoop.yarn.service.utils.FilterUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -64,6 +67,7 @@
import static org.apache.hadoop.yarn.service.component.ComponentEventType.DECOMMISSION_INSTANCE;
import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_AM_CLIENT_PORT_RANGE;
public class ClientAMService extends AbstractService
implements ClientAMProtocol {
@ -84,9 +88,11 @@ public ClientAMService(ServiceContext context) {
@Override protected void serviceStart() throws Exception {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress address = new InetSocketAddress(0);
String nodeHostString = getNMHostName();
InetSocketAddress address = new InetSocketAddress(nodeHostString, 0);
server = rpc.getServer(ClientAMProtocol.class, this, address, conf,
context.secretManager, 1);
context.secretManager, 1, YARN_SERVICE_AM_CLIENT_PORT_RANGE);
// Enable service authorization?
if (conf.getBoolean(
@ -97,9 +103,6 @@ public ClientAMService(ServiceContext context) {
server.start();
String nodeHostString =
System.getenv(ApplicationConstants.Environment.NM_HOST.name());
bindAddress = NetUtils.createSocketAddrForHost(nodeHostString,
server.getListenerAddress().getPort());
@ -107,6 +110,12 @@ public ClientAMService(ServiceContext context) {
super.serviceStart();
}
@VisibleForTesting
String getNMHostName() throws BadClusterStateException {
return ServiceUtils.mandatoryEnvVariable(
ApplicationConstants.Environment.NM_HOST.name());
}
@Override protected void serviceStop() throws Exception {
if (server != null) {
server.stop();

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
View File

@ -129,7 +129,7 @@ protected void serviceInit(Configuration conf) throws Exception {
DefaultMetricsSystem.initialize("ServiceAppMaster");
context.secretManager = new ClientToAMTokenSecretManager(attemptId, null);
ClientAMService clientAMService = new ClientAMService(context);
ClientAMService clientAMService = createClientAMService();
context.clientAMService = clientAMService;
addService(clientAMService);
@ -143,6 +143,11 @@ protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
}
@VisibleForTesting
protected ClientAMService createClientAMService() {
return new ClientAMService(context);
}
// Record the tokens and use them for launching containers.
// e.g. localization requires the hdfs delegation tokens
@VisibleForTesting

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
View File

@ -49,6 +49,8 @@ public class YarnServiceConf {
public static final long DEFAULT_AM_FAILURES_VALIDITY_INTERVAL = -1;
public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
public static final String YARN_SERVICE_AM_CLIENT_PORT_RANGE =
YARN_SERVICE_PREFIX + "am.client.port-range";
public static final String YARN_QUEUE = "yarn.service.queue";
public static final String DEFAULT_YARN_QUEUE = "default";

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
View File

@ -129,6 +129,16 @@ protected Path getAppDir() {
return path;
}
@Override
protected ClientAMService createClientAMService() {
return new ClientAMService(context) {
@Override
String getNMHostName() {
return "0.0.0.0";
}
};
}
@Override
protected ServiceScheduler createServiceScheduler(ServiceContext context)
throws IOException, YarnException {

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
View File

@ -132,6 +132,7 @@ The above config allows the service AM to be retried a maximum of 10 times.
|yarn.service.rolling-log.include-pattern | Regex expression for including log files by name when aggregating the logs while app is running.|
|yarn.service.rolling-log.exclude-pattern | Regex expression for excluding log files by name when aggregating the logs while app is running. If the log file name matches both include and exclude pattern, this file will be excluded.|
|yarn.service.classpath | Comma separated extra class path parameters for yarn services AM. These path elements will be appended to the end of the YARN service AM classpath. |
|yarn.service.am.client.port-range | Range of ports that the Yarn Service AM can use when binding. Leave blank if you want all possible ports. For example 50000-50050,50100-50200. |
### Component-level configuration properties
Component-level service AM configuration properties can be specified either in the cluster `yarn-site.xml` at the global level (effectively overriding the default values system-wide), specified per service in the `properties` field of the `Configuration` object, or specified per component in the `properties` field of the component's `Configuration` object.