big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6472. Improve Java sandbox regex (gphillips via rkanter)

Browse Source
This commit is contained in:
Robert Kanter 2017-04-28 11:01:50 -07:00
parent 373bb4931f
commit 68e45f554b
3 changed files with 30 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
View File

@ -70,11 +70,12 @@ public void initialize(Configuration conf)
private LinuxContainerRuntime pickContainerRuntime(
Map<String, String> environment){
LinuxContainerRuntime runtime;
if (DockerLinuxContainerRuntime.isDockerContainerRequested(environment)){
//Sandbox checked first to ensure DockerRuntime doesn't circumvent controls
if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()){
runtime = javaSandboxLinuxContainerRuntime;
} else if (DockerLinuxContainerRuntime
.isDockerContainerRequested(environment)){
runtime = dockerLinuxContainerRuntime;
} else if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()) {
runtime = javaSandboxLinuxContainerRuntime;
} else {
runtime = defaultLinuxContainerRuntime;
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
View File

@ -141,8 +141,6 @@ public void initialize(Configuration conf)
this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
initializePolicyDir();
super.initialize(conf);
}
@ -223,6 +221,7 @@ public void prepareContainer(ContainerRuntimeContext ctx)
OutputStream policyOutputStream = null;
try {
String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
initializePolicyDir();
Path policyFilePath = Files.createFile(
Paths.get(policyFileDir.toString(),
@ -368,8 +367,12 @@ static final class NMContainerPolicyUtils{
static final String STRIP_POLICY_FLAG = POLICY_APPEND_FLAG + "[^ ]+";
static final String CONTAINS_JAVA_CMD = "\\$" + JAVA_HOME + JAVA_CMD + ".*";
static final String CHAINED_COMMAND_REGEX =
"^.*(&&.+$)|(\\|\\|.+$).*$"; //Matches any occurrences of '||' or '&&'
static final String MULTI_COMMAND_REGEX =
"(?s).*(" + //command read as single line
"(&[^>]|&&)|(\\|{1,2})|(\\|&)|" + //Matches '&','&&','|','||' and '|&'
"(`[^`]+`)|(\\$\\([^)]+\\))|" + //Matches occurrences of $() or ``
"(;)" + //Matches end of statement ';'
").*";
static final String CLEAN_CMD_REGEX =
"(" + SECURITY_FLAG + ")|" +
"(" + STRIP_POLICY_FLAG + ")";
@ -459,7 +462,7 @@ static void appendSecurityFlags(List<String> commands,
String command = commands.get(i);
if(validateJavaHome(env.get(JAVA_HOME.name()))
&& command.matches(CONTAINS_JAVA_CMD)
&& !command.matches(CHAINED_COMMAND_REGEX)){
&& !command.matches(MULTI_COMMAND_REGEX)){
command = command.replaceAll(CLEAN_CMD_REGEX, "");
String securityString = JVM_SECURITY_CMD + policyPath + " ";
if(LOG.isDebugEnabled()) {

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
View File

@ -47,7 +47,8 @@
import java.util.Map;
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CHAINED_COMMAND_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.LOG;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.MULTI_COMMAND_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
@ -293,8 +294,21 @@ public void testDeniedWhitelistGroup() throws ContainerExecutionException {
@Test
public void testChainedCmdRegex(){
Assert.assertTrue("cmd1 && cmd2 || cmd3".matches(CHAINED_COMMAND_REGEX));
Assert.assertFalse("cmd1 &> logfile".matches(CHAINED_COMMAND_REGEX));
String[] multiCmds = {
"cmd1 && cmd2",
"cmd1 || cmd2",
"cmd1 `cmd2`",
"cmd1 $(cmd2)",
"cmd1; \\\n cmd2",
"cmd1; cmd2",
"cmd1|&cmd2",
"cmd1|cmd2",
"cmd1&cmd2"
};
Arrays.stream(multiCmds)
.forEach(cmd -> Assert.assertTrue(cmd.matches(MULTI_COMMAND_REGEX)));
Assert.assertFalse("cmd1 &> logfile".matches(MULTI_COMMAND_REGEX));
}
@Test