big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He

Browse Source
This commit is contained in:
Billie Rinaldi 2017-09-21 10:18:42 -07:00 committed by Jian He
parent 37c9b7327d
commit ce74e64363
12 changed files with 53 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
View File

@ -344,11 +344,7 @@ private void registerServiceInstance(ApplicationAttemptId attemptId,
attemptId.getApplicationId().toString());
serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
PersistencePolicies.APPLICATION);
serviceRecord.description = "Yarn Service Master";
serviceRecord.addExternalEndpoint(RegistryTypeUtils
.ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
context.clientAMService.getBindAddress()));
serviceRecord.description = "YarnServiceMaster";
// set any provided attributes
setUserProvidedServiceRecordAttributes(service.getConfiguration(),

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
View File

@ -170,6 +170,9 @@ private Service loadAppJsonFromLocalFS(
if (!StringUtils.isEmpty(args.getServiceName())) {
service.setName(args.getServiceName());
}
if (!StringUtils.isEmpty(args.queue)) {
service.setQueue(args.queue);
}
return service;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
View File

@ -37,7 +37,7 @@ public File getFile() {
}
@Parameter(names = {
ARG_QUEUE }, description = "Queue to submit the service")
ARG_QUEUE, ARG_SHORT_QUEUE}, description = "Queue to submit the service")
public String queue;
@Parameter(names = {

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
View File

@ -77,6 +77,7 @@ public interface Arguments {
String ARG_PATH = "--path";
String ARG_PRINCIPAL = "--principal";
String ARG_QUEUE = "--queue";
String ARG_SHORT_QUEUE = "-q";
String ARG_LIFETIME = "--lifetime";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCE_MANAGER = "--rm";

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
View File

@ -59,6 +59,7 @@
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import static org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes.*;
import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*;
@ -356,12 +357,11 @@ private void updateServiceRecord(
YarnRegistryViewForProviders yarnRegistry, ContainerStatus status) {
ServiceRecord record = new ServiceRecord();
String containerId = status.getContainerId().toString();
record.set(YarnRegistryAttributes.YARN_ID, containerId);
record.set(YARN_ID, containerId);
record.description = getCompInstanceName();
record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
PersistencePolicies.CONTAINER);
record.set("yarn:ip", status.getIPs());
record.set("yarn:hostname", status.getHost());
record.set(YARN_PERSISTENCE, PersistencePolicies.CONTAINER);
record.set(YARN_IP, status.getIPs().get(0));
record.set(YARN_HOSTNAME, status.getHost());
try {
yarnRegistry
.putComponent(RegistryPathUtils.encodeYarnID(containerId), record);

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.service.provider;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.service.api.records.Service;
@ -91,13 +92,16 @@ public void buildContainerLaunchContext(AbstractLauncher launcher,
component, tokensForSubstitution, instance, context);
// substitute launch command
String launchCommand = ProviderUtils
.substituteStrWithTokens(component.getLaunchCommand(),
tokensForSubstitution);
CommandLineBuilder operation = new CommandLineBuilder();
operation.add(launchCommand);
operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
launcher.addCommand(operation.build());
String launchCommand = component.getLaunchCommand();
// docker container may have empty commands
if (!StringUtils.isEmpty(launchCommand)) {
launchCommand = ProviderUtils
.substituteStrWithTokens(launchCommand, tokensForSubstitution);
CommandLineBuilder operation = new CommandLineBuilder();
operation.add(launchCommand);
operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
launcher.addCommand(operation.build());
}
// By default retry forever every 30 seconds
launcher.setRetryContext(YarnServiceConf

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java
View File

@ -22,7 +22,6 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
import static org.apache.hadoop.registry.client.types.AddressTypes.*;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
import org.apache.hadoop.registry.client.types.ServiceRecord;
@ -36,6 +35,8 @@
import java.util.List;
import java.util.Map;
import static org.apache.hadoop.registry.client.types.AddressTypes.*;
/**
* Static methods to work with registry types —primarily endpoints and the
* list representation of addresses.

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
View File

@ -18,6 +18,8 @@
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xbill.DNS.Name;
import org.xbill.DNS.Type;
@ -32,7 +34,8 @@
*/
public class ApplicationServiceRecordProcessor extends
BaseServiceRecordProcessor {
private static final Logger LOG =
LoggerFactory.getLogger(ApplicationServiceRecordProcessor.class);
/**
* Create an application service record processor.
*
@ -57,6 +60,10 @@ public ApplicationServiceRecordProcessor(
*/
@Override public void initTypeToInfoMapping(ServiceRecord serviceRecord)
throws Exception {
if (serviceRecord.external.isEmpty()) {
LOG.info(serviceRecord.description + ": No external endpoints defined.");
return;
}
for (int type : getRecordTypes()) {
switch (type) {
case Type.A:
@ -309,6 +316,9 @@ public AApplicationRecordDescriptor(String path,
throws Exception {
this.setNames(new Name[] {getServiceName()});
List<Endpoint> endpoints = serviceRecord.external;
if (endpoints.isEmpty()) {
return;
}
// TODO: do we need a "hostname" attribute for an application record or
// can we rely on the first endpoint record.
this.setTarget(InetAddress.getByName(
@ -342,6 +352,9 @@ public AAAAApplicationRecordDescriptor(String path,
@Override protected void init(ServiceRecord serviceRecord)
throws Exception {
super.init(serviceRecord);
if (getTarget() == null) {
return;
}
try {
this.setTarget(getIpv6Address(getTarget()));
} catch (UnknownHostException e) {

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
View File

@ -52,8 +52,8 @@ public abstract class BaseServiceRecordProcessor
private String domain;
private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
private static final String SLIDER_API_PREFIX =
"classpath:org.apache.slider.";
private static final String YARN_SERVICE_API_PREFIX =
"classpath:org.apache.hadoop.yarn.service.";
private static final String HTTP_API_TYPE = "http://";
/**
@ -425,8 +425,8 @@ protected List<String> getTextRecords(Endpoint endpoint) {
*/
protected String getDNSApiFragment(String api) {
String dnsApi = null;
if (api.startsWith(SLIDER_API_PREFIX)) {
dnsApi = api.substring(SLIDER_API_PREFIX.length());
if (api.startsWith(YARN_SERVICE_API_PREFIX)) {
dnsApi = api.substring(YARN_SERVICE_API_PREFIX.length());
} else if (api.startsWith(HTTP_API_TYPE)) {
dnsApi = "http";
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
View File

@ -69,7 +69,8 @@ public class TestRegistryDNS extends Assert {
+ " \"type\" : \"JSONServiceRecord\",\n"
+ " \"description\" : \"Slider Application Master\",\n"
+ " \"external\" : [ {\n"
+ " \"api\" : \"classpath:org.apache.slider.appmaster.ipc\",\n"
+ " \"api\" : \"classpath:org.apache.hadoop.yarn.service.appmaster.ipc"
+ "\",\n"
+ " \"addressType\" : \"host/port\",\n"
+ " \"protocolType\" : \"hadoop/IPC\",\n"
+ " \"addresses\" : [ {\n"
@ -84,7 +85,8 @@ public class TestRegistryDNS extends Assert {
+ " \"uri\" : \"http://192.168.1.5:1027\"\n"
+ " } ]\n"
+ " }, {\n"
+ " \"api\" : \"classpath:org.apache.slider.management\",\n"
+ " \"api\" : \"classpath:org.apache.hadoop.yarn.service.management\""
+ ",\n"
+ " \"addressType\" : \"uri\",\n"
+ " \"protocolType\" : \"REST\",\n"
+ " \"addresses\" : [ {\n"
@ -92,14 +94,16 @@ public class TestRegistryDNS extends Assert {
+ " } ]\n"
+ " } ],\n"
+ " \"internal\" : [ {\n"
+ " \"api\" : \"classpath:org.apache.slider.agents.secure\",\n"
+ " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.secure"
+ "\",\n"
+ " \"addressType\" : \"uri\",\n"
+ " \"protocolType\" : \"REST\",\n"
+ " \"addresses\" : [ {\n"
+ " \"uri\" : \"https://192.168.1.5:47700/ws/v1/slider/agents\"\n"
+ " } ]\n"
+ " }, {\n"
+ " \"api\" : \"classpath:org.apache.slider.agents.oneway\",\n"
+ " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.oneway"
+ "\",\n"
+ " \"addressType\" : \"uri\",\n"
+ " \"protocolType\" : \"REST\",\n"
+ " \"addresses\" : [ {\n"

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
View File

@ -812,13 +812,13 @@ public String[] getIpAndHost(Container container) {
.executePrivilegedOperation(null, privOp, null,
null, true, false);
LOG.info("Docker inspect output for " + containerId + ": " + output);
// strip off quotes if any
output = output.replaceAll("['\"]", "");
int index = output.lastIndexOf(',');
if (index == -1) {
LOG.error("Incorrect format for ip and host");
return null;
}
// strip off quotes if any
output = output.replaceAll("['\"]", "");
String ips = output.substring(0, index).trim();
String host = output.substring(index+1).trim();
String[] ipAndHost = new String[2];

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
View File

@ -92,6 +92,7 @@ Usage `yarn service [sub-command] [service-name] [options]`
Options:
--file,-f The local path to the service definition file.
--queue,-q The queue to which the service is submitted.
--example,-e The name of the example service such as:
Sleeper A simple service that launches a few non-docker sleep containers on YARN.
```