YARN-6472. Improve Java sandbox regex (gphillips via rkanter)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java

@@ -70,11 +70,12 @@ public void initialize(Configuration conf)
   private LinuxContainerRuntime pickContainerRuntime(
       Map<String, String> environment){
     LinuxContainerRuntime runtime;
-
+    //Sandbox checked first to ensure DockerRuntime doesn't circumvent controls
-    if (DockerLinuxContainerRuntime.isDockerContainerRequested(environment)){
+    if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()){
+        runtime = javaSandboxLinuxContainerRuntime;
+    } else if (DockerLinuxContainerRuntime
+        .isDockerContainerRequested(environment)){
       runtime = dockerLinuxContainerRuntime;
-    } else if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()) {
-      runtime = javaSandboxLinuxContainerRuntime;
     } else {
       runtime = defaultLinuxContainerRuntime;
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java

@@ -141,8 +141,6 @@ public void initialize(Configuration conf)
             this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
                 YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
 
-    initializePolicyDir();
-
     super.initialize(conf);
   }
 
@@ -223,6 +221,7 @@ public void prepareContainer(ContainerRuntimeContext ctx)
       OutputStream policyOutputStream = null;
       try {
         String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
+        initializePolicyDir();
 
         Path policyFilePath = Files.createFile(
             Paths.get(policyFileDir.toString(),
@@ -368,8 +367,12 @@ static final class NMContainerPolicyUtils{
 
     static final String STRIP_POLICY_FLAG = POLICY_APPEND_FLAG + "[^ ]+";
     static final String CONTAINS_JAVA_CMD = "\\$" + JAVA_HOME + JAVA_CMD + ".*";
-    static final String CHAINED_COMMAND_REGEX =
+    static final String MULTI_COMMAND_REGEX =
-        "^.*(&&.+$)|(\\|\\|.+$).*$";  //Matches any occurrences of '||' or '&&'
+        "(?s).*(" + //command read as single line
+        "(&[^>]|&&)|(\\|{1,2})|(\\|&)|" + //Matches '&','&&','|','||' and '|&'
+        "(`[^`]+`)|(\\$\\([^)]+\\))|" + //Matches occurrences of $() or ``
+        "(;)" + //Matches end of statement ';'
+        ").*";
     static final String CLEAN_CMD_REGEX =
         "(" + SECURITY_FLAG + ")|" +
             "(" + STRIP_POLICY_FLAG + ")";
@@ -459,7 +462,7 @@ static void appendSecurityFlags(List<String> commands,
         String command = commands.get(i);
         if(validateJavaHome(env.get(JAVA_HOME.name()))
             && command.matches(CONTAINS_JAVA_CMD)
-            && !command.matches(CHAINED_COMMAND_REGEX)){
+            && !command.matches(MULTI_COMMAND_REGEX)){
           command = command.replaceAll(CLEAN_CMD_REGEX, "");
           String securityString = JVM_SECURITY_CMD + policyPath + " ";
           if(LOG.isDebugEnabled()) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java

@@ -47,7 +47,8 @@
 import java.util.Map;
 
 import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
-import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CHAINED_COMMAND_REGEX;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.LOG;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.MULTI_COMMAND_REGEX;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
@@ -293,8 +294,21 @@ public void testDeniedWhitelistGroup() throws ContainerExecutionException {
 
   @Test
   public void testChainedCmdRegex(){
-    Assert.assertTrue("cmd1 && cmd2 || cmd3".matches(CHAINED_COMMAND_REGEX));
+    String[] multiCmds = {
-    Assert.assertFalse("cmd1 &> logfile".matches(CHAINED_COMMAND_REGEX));
+        "cmd1 && cmd2",
+        "cmd1 || cmd2",
+        "cmd1 `cmd2`",
+        "cmd1 $(cmd2)",
+        "cmd1; \\\n cmd2",
+        "cmd1; cmd2",
+        "cmd1|&cmd2",
+        "cmd1|cmd2",
+        "cmd1&cmd2"
+    };
+
+    Arrays.stream(multiCmds)
+        .forEach(cmd -> Assert.assertTrue(cmd.matches(MULTI_COMMAND_REGEX)));
+    Assert.assertFalse("cmd1 &> logfile".matches(MULTI_COMMAND_REGEX));
   }
 
   @Test