YARN-6804. [yarn-native-services changes] Allow custom hostname for docker containers in native services. Contributed by Billie Rinaldi

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java

@@ -81,6 +81,7 @@ public abstract class AbstractLauncher extends Configured {
   protected boolean yarnDockerMode = false;
   protected String dockerImage;
   protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
+  protected String dockerHostname;
   protected String yarnContainerMountPoints;
   protected String runPrivilegedContainer;
 
@@ -236,6 +237,8 @@ public ContainerLaunchContext completeContainerLaunch() throws IOException {
       env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
       env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);
       env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", dockerNetwork);
+      env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME",
+          dockerHostname);
       env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
       StringBuilder sb = new StringBuilder();
       for (Entry<String,String> mount : mountPaths.entrySet()) {
@@ -469,6 +472,10 @@ public void setDockerNetwork(String dockerNetwork) {
     this.dockerNetwork = dockerNetwork;
   }
 
+  public void setDockerHostname(String dockerHostname) {
+    this.dockerHostname = dockerHostname;
+  }
+
   public void setYarnContainerMountPoints(String yarnContainerMountPoints) {
     this.yarnContainerMountPoints = yarnContainerMountPoints;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java

@@ -60,8 +60,9 @@ protected AbstractProviderService(String name) {
     super(name);
   }
 
-  public abstract void processArtifact(ContainerLauncher launcher, Component
+  public abstract void processArtifact(ContainerLauncher launcher,
-      component, SliderFileSystem fileSystem) throws IOException;
+      Application application, RoleInstance roleInstance,
+      SliderFileSystem fileSystem) throws IOException;
 
   @Override
   public void setAMState(StateAccessForProviders stateAccessor) {
@@ -78,7 +79,7 @@ public void buildContainerLaunchContext(ContainerLauncher launcher,
       SliderFileSystem fileSystem, RoleInstance roleInstance)
       throws IOException, SliderException {
     Component component = providerRole.component;
-    processArtifact(launcher, component, fileSystem);
+    processArtifact(launcher, application, roleInstance, fileSystem);
 
     // Generate tokens (key-value pair) for config substitution.
     // Get pre-defined tokens

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java

@@ -17,10 +17,10 @@
  */
 package org.apache.slider.providers;
 
-import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.Application;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.server.appmaster.state.RoleInstance;
 
 import java.io.IOException;
 
@@ -31,7 +31,8 @@ protected DefaultProviderService() {
   }
 
   @Override
-  public void processArtifact(ContainerLauncher launcher, Component
+  public void processArtifact(ContainerLauncher launcher, Application
-      component, SliderFileSystem fileSystem) throws IOException {
+      application, RoleInstance roleInstance, SliderFileSystem fileSystem)
+      throws IOException {
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java

@@ -504,8 +504,7 @@ public void updateServiceRecord(StateAccessForProviders amState,
         // create and publish updated service record (including hostname & ip)
         ServiceRecord record = new ServiceRecord();
         record.set(YarnRegistryAttributes.YARN_ID, containerId);
-        String componentInstanceName = role.getCompInstanceName();
+        record.description = role.getCompInstanceName();
-        record.description = componentInstanceName.replaceAll("_", "-");
         record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
             PersistencePolicies.CONTAINER);
         // TODO: use constants from YarnRegistryAttributes

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java

@@ -17,14 +17,19 @@
  */
 package org.apache.slider.providers.docker;
 
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.slider.api.resource.Application;
 import org.apache.slider.api.resource.Component;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.core.launch.ContainerLauncher;
 import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.server.appmaster.state.RoleInstance;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.text.MessageFormat;
 
 public class DockerProviderService extends AbstractProviderService
     implements DockerKeys {
@@ -36,12 +41,26 @@ protected DockerProviderService() {
     super(DockerProviderService.class.getSimpleName());
   }
 
-  public void processArtifact(ContainerLauncher launcher, Component
+  public void processArtifact(ContainerLauncher launcher, Application
-      component, SliderFileSystem fileSystem) throws IOException {
+      application, RoleInstance roleInstance, SliderFileSystem fileSystem)
+      throws IOException {
+    Component component = roleInstance.providerRole.component;
     launcher.setYarnDockerMode(true);
     launcher.setDockerImage(component.getArtifact().getId());
     launcher.setDockerNetwork(component.getConfiguration()
         .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
+    String domain = getConfig().get(RegistryConstants.KEY_DNS_DOMAIN);
+    String hostname;
+    if (domain == null || domain.isEmpty()) {
+      hostname = MessageFormat.format("{0}.{1}.{2}", roleInstance
+          .getCompInstanceName(), application.getName(), RegistryUtils
+          .currentUser());
+    } else {
+      hostname = MessageFormat.format("{0}.{1}.{2}.{3}", roleInstance
+          .getCompInstanceName(), application.getName(), RegistryUtils
+          .currentUser(), domain);
+    }
+    launcher.setDockerHostname(hostname);
     launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer());
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java

@@ -20,10 +20,12 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.slider.api.resource.Application;
 import org.apache.slider.api.resource.Component;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.core.launch.ContainerLauncher;
 import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.server.appmaster.state.RoleInstance;
 
 import java.io.IOException;
 
@@ -34,9 +36,11 @@ protected TarballProviderService() {
   }
 
   @Override
-  public void processArtifact(ContainerLauncher launcher, Component
+  public void processArtifact(ContainerLauncher launcher, Application
-      component, SliderFileSystem fileSystem) throws IOException {
+      application, RoleInstance roleInstance, SliderFileSystem fileSystem)
-    Path artifact =  new Path(component.getArtifact().getId());
+      throws IOException {
+    Path artifact = new Path(roleInstance.providerRole.component
+        .getArtifact().getId());
     if (!fileSystem.isFile(artifact)) {
       throw new IOException("Package doesn't exist as a resource: " +
           artifact.toString());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java

@@ -125,6 +125,7 @@ public RoleInstance(Container container, ProviderRole role) {
     } else {
       compInstanceName = role.name;
     }
+    compInstanceName = compInstanceName.toLowerCase().replaceAll("_", "-");
     this.providerRole = role;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java

@@ -285,7 +285,6 @@ public ContainerRecordDescriptor(String path, ServiceRecord record)
      */
     protected Name getContainerIDName() throws TextParseException {
       String containerID = RegistryPathUtils.lastPathEntry(getPath());
-      containerID = containerID.replace("container", "ctr");
       return Name.fromString(String.format("%s.%s", containerID, domain));
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java

@@ -16,6 +16,7 @@
  */
 package org.apache.hadoop.registry.server.dns;
 
+import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
 import org.xbill.DNS.Name;
@@ -156,9 +157,11 @@ public TXTContainerRecordDescriptor(String path,
      */
     @Override protected void init(ServiceRecord serviceRecord) {
       try {
-        this.setNames(new Name[] {getContainerIDName()});
+        this.setNames(new Name[] {getContainerName()});
       } catch (TextParseException e) {
         // log
+      } catch (PathNotFoundException e) {
+        // log
       }
       List<String> txts = new ArrayList<>();
       txts.add("id=" + serviceRecord.get(YarnRegistryAttributes.YARN_ID));
@@ -200,9 +203,11 @@ public PTRContainerRecordDescriptor(String path,
       }
       this.setNames(new Name[] {reverseLookupName});
       try {
-        this.setTarget(getContainerIDName());
+        this.setTarget(getContainerName());
       } catch (TextParseException e) {
         //LOG
+      } catch (PathNotFoundException e) {
+        //LOG
       }
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java

@@ -111,7 +111,7 @@ public class TestRegistryDNS extends Assert {
       + "}\n";
   static final String CONTAINER_RECORD = "{\n"
       + "  \"type\" : \"JSONServiceRecord\",\n"
-      + "  \"description\" : \"YCLOUD\",\n"
+      + "  \"description\" : \"COMP-NAME\",\n"
       + "  \"external\" : [ ],\n"
       + "  \"internal\" : [ ],\n"
       + "  \"yarn:id\" : \"container_e50_1451931954322_0016_01_000002\",\n"
@@ -122,7 +122,7 @@ public class TestRegistryDNS extends Assert {
 
   private static final String CONTAINER_RECORD_NO_IP = "{\n"
       + "  \"type\" : \"JSONServiceRecord\",\n"
-      + "  \"description\" : \"YCLOUD\",\n"
+      + "  \"description\" : \"COMP-NAME\",\n"
       + "  \"external\" : [ ],\n"
       + "  \"internal\" : [ ],\n"
       + "  \"yarn:id\" : \"container_e50_1451931954322_0016_01_000002\",\n"
@@ -131,7 +131,7 @@ public class TestRegistryDNS extends Assert {
 
   private static final String CONTAINER_RECORD_YARN_PERSISTANCE_ABSENT = "{\n"
       + "  \"type\" : \"JSONServiceRecord\",\n"
-      + "  \"description\" : \"YCLOUD\",\n"
+      + "  \"description\" : \"COMP-NAME\",\n"
       + "  \"external\" : [ ],\n"
       + "  \"internal\" : [ ],\n"
       + "  \"yarn:id\" : \"container_e50_1451931954322_0016_01_000003\",\n"
@@ -216,7 +216,7 @@ public void testContainerRegistration() throws Exception {
         CONTAINER_RECORD.getBytes());
     getRegistryDNS().register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000002",
+            + "ctr-e50-1451931954322-0016-01-000002",
         record);
 
     // start assessing whether correct records are available
@@ -225,7 +225,7 @@ public void testContainerRegistration() throws Exception {
     assertEquals("wrong result", "172.17.0.19",
         ((ARecord) recs[0]).getAddress().getHostAddress());
 
-    recs = assertDNSQuery("ycloud.test1.root.hwx.test.", 1);
+    recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1);
     assertTrue("not an ARecord", recs[0] instanceof ARecord);
   }
 
@@ -235,7 +235,7 @@ public void testContainerRegistrationPersistanceAbsent() throws Exception {
         CONTAINER_RECORD_YARN_PERSISTANCE_ABSENT.getBytes());
     registryDNS.register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000003",
+            + "ctr-e50-1451931954322-0016-01-000003",
          record);
 
     Name name =
@@ -254,7 +254,7 @@ public void testRecordTTL() throws Exception {
         CONTAINER_RECORD.getBytes());
     getRegistryDNS().register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000002",
+            + "ctr-e50-1451931954322-0016-01-000002",
         record);
 
     // start assessing whether correct records are available
@@ -264,7 +264,7 @@ public void testRecordTTL() throws Exception {
         ((ARecord) recs[0]).getAddress().getHostAddress());
     assertEquals("wrong ttl", 30L, recs[0].getTTL());
 
-    recs = assertDNSQuery("ycloud.test1.root.hwx.test.", 1);
+    recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1);
     assertTrue("not an ARecord", recs[0] instanceof ARecord);
 
     assertEquals("wrong ttl", 30L, recs[0].getTTL());
@@ -276,13 +276,13 @@ public void testReverseLookup() throws Exception {
         CONTAINER_RECORD.getBytes());
     getRegistryDNS().register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000002",
+            + "ctr-e50-1451931954322-0016-01-000002",
         record);
 
     // start assessing whether correct records are available
     Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
     assertEquals("wrong result",
-        "ctr-e50-1451931954322-0016-01-000002.hwx.test.",
+        "comp-name.test1.root.hwx.test.",
         ((PTRRecord) recs[0]).getTarget().toString());
   }
 
@@ -302,13 +302,13 @@ public void testReverseLookupInLargeNetwork() throws Exception {
         CONTAINER_RECORD.getBytes());
     getRegistryDNS().register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000002",
+            + "ctr-e50-1451931954322-0016-01-000002",
         record);
 
     // start assessing whether correct records are available
     Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
     assertEquals("wrong result",
-        "ctr-e50-1451931954322-0016-01-000002.hwx.test.",
+        "comp-name.test1.root.hwx.test.",
         ((PTRRecord) recs[0]).getTarget().toString());
   }
 
@@ -318,7 +318,7 @@ public void testMissingReverseLookup() throws Exception {
         CONTAINER_RECORD.getBytes());
     getRegistryDNS().register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000002",
+            + "ctr-e50-1451931954322-0016-01-000002",
         record);
 
     // start assessing whether correct records are available
@@ -339,7 +339,7 @@ public void testNoContainerIP() throws Exception {
         CONTAINER_RECORD_NO_IP.getBytes());
     getRegistryDNS().register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000002",
+            + "ctr-e50-1451931954322-0016-01-000002",
         record);
 
     // start assessing whether correct records are available
@@ -453,7 +453,7 @@ public void testAAAALookup() throws Exception {
         CONTAINER_RECORD.getBytes());
     getRegistryDNS().register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000002",
+            + "ctr-e50-1451931954322-0016-01-000002",
         record);
 
     // start assessing whether correct records are available
@@ -462,7 +462,7 @@ public void testAAAALookup() throws Exception {
     assertEquals("wrong result", "172.17.0.19",
         ((AAAARecord) recs[0]).getAddress().getHostAddress());
 
-    recs = assertDNSQuery("ycloud.test1.root.hwx.test.", Type.AAAA, 1);
+    recs = assertDNSQuery("comp-name.test1.root.hwx.test.", Type.AAAA, 1);
     assertTrue("not an ARecord", recs[0] instanceof AAAARecord);
   }
 
@@ -472,7 +472,7 @@ public void testNegativeLookup() throws Exception {
         CONTAINER_RECORD.getBytes());
     getRegistryDNS().register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000002",
+            + "ctr-e50-1451931954322-0016-01-000002",
         record);
 
     // start assessing whether correct records are available
@@ -528,7 +528,7 @@ public void testReadMasterFile() throws Exception {
         CONTAINER_RECORD.getBytes());
     getRegistryDNS().register(
         "/registry/users/root/services/org-apache-slider/test1/components/"
-            + "container-e50-1451931954322-0016-01-000002",
+            + "ctr-e50-1451931954322-0016-01-000002",
         record);
 
     // start assessing whether correct records are available
@@ -537,13 +537,13 @@ public void testReadMasterFile() throws Exception {
     assertEquals("wrong result", "172.17.0.19",
         ((ARecord) recs[0]).getAddress().getHostAddress());
 
-    recs = assertDNSQuery("ycloud.test1.root.hwx.test.", 1);
+    recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1);
     assertTrue("not an ARecord", recs[0] instanceof ARecord);
 
     // lookup dyanmic reverse records
     recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
     assertEquals("wrong result",
-        "ctr-e50-1451931954322-0016-01-000002.hwx.test.",
+        "comp-name.test1.root.hwx.test.",
         ((PTRRecord) recs[0]).getTarget().toString());
 
     // now lookup static reverse records

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -31,6 +31,7 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
@@ -398,6 +399,11 @@ private void setHostname(DockerRunCommand runCommand, String
       throws ContainerExecutionException {
     if (name == null || name.isEmpty()) {
       name = RegistryPathUtils.encodeYarnID(containerIdStr);
+
+      String domain = conf.get(RegistryConstants.KEY_DNS_DOMAIN);
+      if (domain != null) {
+        name += ("." + domain);
+      }
       validateHostname(name);
     }