big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8805. Automatically convert the launch command to the exec form when using entrypoint support

Browse Source
This commit is contained in:
Eric Badger 2019-03-11 12:43:14 -05:00
parent b4aa24d3c5
commit fa7a0b269a
4 changed files with 55 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
View File

@ -47,9 +47,12 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.COMPONENT_ID;
@ -149,6 +152,20 @@ public static String substituteStrWithTokens(String content,
return content;
}
public static String replaceSpacesWithDelimiter(String content,
String delimiter) {
List<String> parts = new ArrayList<String>();
Matcher m = Pattern.compile("([^\"]\\S*|\".+?\")\\s*").matcher(content);
while (m.find()) {
String part = m.group(1);
if(part.startsWith("\"") && part.endsWith("\"")) {
part = part.replaceAll("^\"|\"$", "");
}
parts.add(part);
}
return String.join(delimiter, parts);
}
// configs will be substituted by corresponding env in tokenMap
public static void substituteMapWithTokens(Map<String, String> configs,
Map<String, String> tokenMap) {

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
View File

@ -25,6 +25,7 @@
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
@ -84,6 +85,11 @@ public void buildContainerLaunchCommand(AbstractLauncher launcher,
if (useEntryPoint) {
String launchCommand = compLaunchContext.getLaunchCommand();
if (!StringUtils.isEmpty(launchCommand)) {
if(launchCommand.contains(" ")) {
// convert space delimiter command to exec format
launchCommand = ProviderUtils
.replaceSpacesWithDelimiter(launchCommand, ",");
}
launcher.addCommand(launchCommand);
}
} else {

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
View File

@ -82,7 +82,7 @@ public void testBuildContainerLaunchCommand() throws Exception {
Component component = serviceContext.scheduler.getAllComponents().entrySet()
.iterator().next().getValue();
ContainerLaunchService.ComponentLaunchContext clc =
createEntryPointCLCFor(testService, component);
createEntryPointCLCFor(testService, component, "sleep,9000");
ComponentInstance instance = component.getAllComponentInstances().iterator()
.next();
@ -95,13 +95,32 @@ public void testBuildContainerLaunchCommand() throws Exception {
launcher.getCommands());
}
@Test
public void testBuildContainerLaunchCommandWithSpace() throws Exception {
AbstractProviderService providerService = new DockerProviderService();
Component component = serviceContext.scheduler.getAllComponents().entrySet()
.iterator().next().getValue();
ContainerLaunchService.ComponentLaunchContext clc =
createEntryPointCLCFor(testService, component, "ls -l \" space\"");
ComponentInstance instance = component.getAllComponentInstances().iterator()
.next();
Container container = mock(Container.class);
providerService.buildContainerLaunchCommand(launcher, testService, instance,
rule.getFs(), serviceContext.scheduler.getConfig(), container, clc,
null);
Assert.assertEquals("commands don't match.",
Lists.newArrayList("ls,-l, space"), launcher.getCommands());
}
@Test
public void testBuildContainerLaunchContext() throws Exception {
AbstractProviderService providerService = new DockerProviderService();
Component component = serviceContext.scheduler.getAllComponents().entrySet()
.iterator().next().getValue();
ContainerLaunchService.ComponentLaunchContext clc =
createEntryPointCLCFor(testService, component);
createEntryPointCLCFor(testService, component, "sleep,9000");
ComponentInstance instance = component.getAllComponentInstances().iterator()
.next();
@ -118,8 +137,8 @@ public void testBuildContainerLaunchContext() throws Exception {
}
private static ContainerLaunchService.ComponentLaunchContext
createEntryPointCLCFor(Service service, Component component) {
String launchCmd = "sleep,9000";
createEntryPointCLCFor(Service service, Component component,
String launchCmd) {
Artifact artifact = new Artifact();
artifact.setType(Artifact.TypeEnum.DOCKER);
artifact.setId("example");

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
View File

@ -168,4 +168,13 @@ public void setShouldBeUploadedToSharedCache(
Assert.assertEquals(resolved.getResolvedRsrcPaths().get("destFile1"),
"destFile1");
}
@Test
public void testReplaceSpacesWithDelimiter() {
String command = "ls -l \" space\"";
String expected = "ls,-l, space";
String actual = ProviderUtils.replaceSpacesWithDelimiter(command, ",");
Assert.assertEquals("replaceSpaceWithDelimiter produces unexpected result.",
expected, actual);
}
}