YARN-6447. Provide container sandbox policies for groups (gphillips via rkanter)
This commit is contained in:
Robert Kanter
parent
101852ca11
commit
18c494a00c
@ -1500,6 +1500,10 @@ public static boolean isAclEnabled(Configuration conf) {
|
||||
public static final String YARN_CONTAINER_SANDBOX_POLICY =
|
||||
YARN_CONTAINER_SANDBOX + ".policy";
|
||||
|
||||
/** Prefix for group to policy file mapping.*/
|
||||
public static final String YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX =
|
||||
YARN_CONTAINER_SANDBOX_POLICY + ".group.";
|
||||
|
||||
/** The group which will run by default without the java security manager.*/
|
||||
public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP =
|
||||
YARN_CONTAINER_SANDBOX + ".whitelist-group";
|
||||
|
||||
@ -50,10 +50,13 @@
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.apache.hadoop.fs.Path.SEPARATOR;
|
||||
import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR;
|
||||
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
|
||||
import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX;
|
||||
import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
|
||||
@ -93,13 +96,21 @@
|
||||
* Accepts canonical path to a java policy file on the local filesystem.
|
||||
* This file will be loaded as the base policy, any additional container
|
||||
* grants will be appended to this base file. If not specified, the default
|
||||
* java.policy file provided with hadoop resources will be used.
|
||||
* java.policy file provided with hadoop resources will be used.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} :
|
||||
* Optional setting to specify a YARN queue which will be exempt from the
|
||||
* sand-boxing process.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@value
|
||||
* YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX}$groupName :
|
||||
* Optional setting to map groups to java policy files. The value is a path
|
||||
* to the java policy file for $groupName. A user which is a member of
|
||||
* multiple groups with different policies will receive the superset of all
|
||||
* the permissions across their groups.
|
||||
* </li>
|
||||
* </ul>
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
@ -138,7 +149,7 @@ public void initialize(Configuration conf)
|
||||
this.configuration = conf;
|
||||
this.sandboxMode =
|
||||
SandboxMode.get(
|
||||
this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
|
||||
this.configuration.get(YARN_CONTAINER_SANDBOX,
|
||||
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
|
||||
|
||||
super.initialize(conf);
|
||||
@ -211,8 +222,10 @@ public void prepareContainer(ContainerRuntimeContext ctx)
|
||||
ctx.getExecutionAttribute(CONTAINER_RUN_CMDS);
|
||||
Map<String, String> env =
|
||||
ctx.getContainer().getLaunchContext().getEnvironment();
|
||||
String username =
|
||||
ctx.getExecutionAttribute(USER);
|
||||
|
||||
if(!isSandboxContainerWhitelisted(ctx, commands)) {
|
||||
if(!isSandboxContainerWhitelisted(username, commands)) {
|
||||
String tmpDirBase = configuration.get("hadoop.tmp.dir");
|
||||
if (tmpDirBase == null) {
|
||||
throw new ContainerExecutionException("hadoop.tmp.dir not set!");
|
||||
@ -223,6 +236,8 @@ public void prepareContainer(ContainerRuntimeContext ctx)
|
||||
String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
|
||||
initializePolicyDir();
|
||||
|
||||
List<String> groupPolicyFiles =
|
||||
getGroupPolicyFiles(configuration, ctx.getExecutionAttribute(USER));
|
||||
Path policyFilePath = Files.createFile(
|
||||
Paths.get(policyFileDir.toString(),
|
||||
containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
|
||||
@ -231,12 +246,12 @@ public void prepareContainer(ContainerRuntimeContext ctx)
|
||||
|
||||
containerPolicies.put(containerID, policyFilePath);
|
||||
|
||||
NMContainerPolicyUtils.generatePolicyFile(
|
||||
policyOutputStream, localDirs, resources, configuration);
|
||||
NMContainerPolicyUtils.generatePolicyFile(policyOutputStream,
|
||||
localDirs, groupPolicyFiles, resources, configuration);
|
||||
NMContainerPolicyUtils.appendSecurityFlags(
|
||||
commands, env, policyFilePath, sandboxMode);
|
||||
|
||||
} catch (Exception e) {
|
||||
} catch (IOException e) {
|
||||
throw new ContainerExecutionException(e);
|
||||
} finally {
|
||||
IOUtils.cleanup(LOG, policyOutputStream);
|
||||
@ -264,15 +279,32 @@ boolean isSandboxContainerRequested() {
|
||||
return sandboxMode != SandboxMode.disabled;
|
||||
}
|
||||
|
||||
private static List<String> getGroupPolicyFiles(Configuration conf,
|
||||
String user) throws ContainerExecutionException {
|
||||
Groups groups = Groups.getUserToGroupsMappingService(conf);
|
||||
List<String> userGroups;
|
||||
try {
|
||||
userGroups = groups.getGroups(user);
|
||||
} catch (IOException e) {
|
||||
throw new ContainerExecutionException("Container user does not exist");
|
||||
}
|
||||
|
||||
return userGroups.stream()
|
||||
.map(group -> conf.get(YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX
|
||||
+ group))
|
||||
.filter(groupPolicy -> groupPolicy != null)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if the container should be whitelisted (i.e. exempt from the
|
||||
* Java Security Manager).
|
||||
* @param ctx The container runtime context for the requested container
|
||||
* @param username The name of the user running the container
|
||||
* @param commands The list of run commands for the container
|
||||
* @return boolean value denoting whether the container should be whitelisted.
|
||||
* @throws ContainerExecutionException If container user can not be resolved
|
||||
*/
|
||||
private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
|
||||
private boolean isSandboxContainerWhitelisted(String username,
|
||||
List<String> commands) throws ContainerExecutionException {
|
||||
String whitelistGroup = configuration.get(
|
||||
YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
|
||||
@ -281,7 +313,7 @@ private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
|
||||
boolean isWhitelisted = false;
|
||||
|
||||
try {
|
||||
userGroups = groups.getGroups(ctx.getExecutionAttribute(USER));
|
||||
userGroups = groups.getGroups(username);
|
||||
} catch (IOException e) {
|
||||
throw new ContainerExecutionException("Container user does not exist");
|
||||
}
|
||||
@ -399,8 +431,9 @@ static final class NMContainerPolicyUtils{
|
||||
* base policy file or if it is unable to create a new policy file.
|
||||
*/
|
||||
static void generatePolicyFile(OutputStream policyOutStream,
|
||||
List<String> localDirs, Map<org.apache.hadoop.fs.Path,
|
||||
List<String>> resources, Configuration conf)
|
||||
List<String> localDirs, List<String> groupPolicyPaths,
|
||||
Map<org.apache.hadoop.fs.Path, List<String>> resources,
|
||||
Configuration conf)
|
||||
throws IOException {
|
||||
|
||||
String policyFilePath =
|
||||
@ -414,13 +447,16 @@ static void generatePolicyFile(OutputStream policyOutStream,
|
||||
cacheDirs.add(path.getParent().toString());
|
||||
}
|
||||
|
||||
if(policyFilePath == null) {
|
||||
if (groupPolicyPaths != null) {
|
||||
for(String policyPath : groupPolicyPaths) {
|
||||
Files.copy(Paths.get(policyPath), policyOutStream);
|
||||
}
|
||||
} else if (policyFilePath == null) {
|
||||
IOUtils.copyBytes(
|
||||
NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE),
|
||||
policyOutStream, conf, false);
|
||||
} else {
|
||||
Files.copy(Paths.get(policyFilePath), policyOutStream);
|
||||
policyOutStream.flush();
|
||||
}
|
||||
|
||||
Formatter filePermissionFormat = new Formatter(policyOutStream,
|
||||
@ -484,7 +520,7 @@ static void appendSecurityFlags(List<String> commands,
|
||||
|
||||
private static boolean validateJavaHome(String containerJavaHome)
|
||||
throws ContainerExecutionException{
|
||||
if (System.getenv(JAVA_HOME.name()) == null){
|
||||
if (System.getenv(JAVA_HOME.name()) == null) {
|
||||
throw new ContainerExecutionException(
|
||||
"JAVA_HOME is not set for NodeManager");
|
||||
}
|
||||
|
||||
@ -37,20 +37,29 @@
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FilePermission;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.net.SocketPermission;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.security.Permission;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Formatter;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.LOG;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.MULTI_COMMAND_REGEX;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_APPEND_FLAG;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG;
|
||||
@ -76,8 +85,9 @@
|
||||
*/
|
||||
public class TestJavaSandboxLinuxContainerRuntime {
|
||||
|
||||
private static final String HADOOP_HOME = "hadoop.home.dir";
|
||||
private static String hadoopHomeDir = System.getProperty(HADOOP_HOME);
|
||||
private final static String HADOOP_HOME = "hadoop.home.dir";
|
||||
private final static String HADOOP_HOME_DIR = System.getProperty(HADOOP_HOME);
|
||||
private final Properties baseProps = new Properties(System.getProperties());
|
||||
|
||||
@Rule
|
||||
public ExpectedException exception = ExpectedException.none();
|
||||
@ -101,11 +111,12 @@ public class TestJavaSandboxLinuxContainerRuntime {
|
||||
private final static String WHITELIST_GROUP = "captains";
|
||||
private final static String CONTAINER_ID = "container_1234567890";
|
||||
private final static String APPLICATION_ID = "application_1234567890";
|
||||
private File baseTestDirectory;
|
||||
|
||||
@Before
|
||||
public void setup() throws Exception {
|
||||
|
||||
File baseTestDirectory = new File(System.getProperty("test.build.data",
|
||||
baseTestDirectory = new File(System.getProperty("test.build.data",
|
||||
System.getProperty("java.io.tmpdir", "target")),
|
||||
TestJavaSandboxLinuxContainerRuntime.class.getName());
|
||||
|
||||
@ -114,10 +125,8 @@ public void setup() throws Exception {
|
||||
|
||||
conf = new Configuration();
|
||||
conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
|
||||
WHITELIST_USER + "=" + WHITELIST_GROUP + ";"
|
||||
WHITELIST_USER + "=" + WHITELIST_GROUP + "," + NORMAL_GROUP + ";"
|
||||
+ NORMAL_USER + "=" + NORMAL_GROUP + ";");
|
||||
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
|
||||
WHITELIST_GROUP);
|
||||
conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath());
|
||||
|
||||
Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(),
|
||||
@ -151,21 +160,20 @@ public void setup() throws Exception {
|
||||
|
||||
runtimeContextBuilder = createRuntimeContext();
|
||||
|
||||
if (hadoopHomeDir == null) {
|
||||
if (HADOOP_HOME_DIR == null) {
|
||||
System.setProperty(HADOOP_HOME, policyFile.getParent());
|
||||
}
|
||||
|
||||
OutputStream outStream = new FileOutputStream(policyFile);
|
||||
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
|
||||
.generatePolicyFile(outStream, symLinks, resources, conf);
|
||||
.generatePolicyFile(outStream, symLinks, null, resources, conf);
|
||||
outStream.close();
|
||||
|
||||
System.setProperty("java.security.policy", policyFile.getCanonicalPath());
|
||||
securityManager = new SecurityManager();
|
||||
|
||||
}
|
||||
|
||||
public ContainerRuntimeContext.Builder createRuntimeContext(){
|
||||
public ContainerRuntimeContext.Builder createRuntimeContext(){
|
||||
|
||||
Container container = mock(Container.class);
|
||||
ContainerLaunchContext ctx = mock(ContainerLaunchContext.class);
|
||||
@ -194,6 +202,71 @@ public ContainerRuntimeContext.Builder createRuntimeContext(){
|
||||
return builder;
|
||||
}
|
||||
|
||||
public static final String SOCKET_PERMISSION_FORMAT =
|
||||
"grant { \n" +
|
||||
" permission %1s \"%2s\", \"%3s\";\n" +
|
||||
"};\n";
|
||||
public static final String RUNTIME_PERMISSION_FORMAT =
|
||||
"grant { \n" +
|
||||
" permission %1s \"%2s\";\n" +
|
||||
"};\n";
|
||||
|
||||
@Test
|
||||
public void testGroupPolicies()
|
||||
throws IOException, ContainerExecutionException {
|
||||
// Generate new policy files each containing one grant
|
||||
File openSocketPolicyFile =
|
||||
File.createTempFile("openSocket", "policy", baseTestDirectory);
|
||||
File classLoaderPolicyFile =
|
||||
File.createTempFile("createClassLoader", "policy", baseTestDirectory);
|
||||
Permission socketPerm = new SocketPermission("localhost:0", "listen");
|
||||
Permission runtimePerm = new RuntimePermission("createClassLoader");
|
||||
|
||||
StringBuilder socketPermString = new StringBuilder();
|
||||
Formatter openSocketPolicyFormatter = new Formatter(socketPermString);
|
||||
openSocketPolicyFormatter.format(SOCKET_PERMISSION_FORMAT,
|
||||
socketPerm.getClass().getName(), socketPerm.getName(),
|
||||
socketPerm.getActions());
|
||||
FileWriter socketPermWriter = new FileWriter(openSocketPolicyFile);
|
||||
socketPermWriter.write(socketPermString.toString());
|
||||
socketPermWriter.close();
|
||||
|
||||
StringBuilder classLoaderPermString = new StringBuilder();
|
||||
Formatter classLoaderPolicyFormatter = new Formatter(classLoaderPermString);
|
||||
classLoaderPolicyFormatter.format(RUNTIME_PERMISSION_FORMAT,
|
||||
runtimePerm.getClass().getName(), runtimePerm.getName());
|
||||
FileWriter classLoaderPermWriter = new FileWriter(classLoaderPolicyFile);
|
||||
classLoaderPermWriter.write(classLoaderPermString.toString());
|
||||
classLoaderPermWriter.close();
|
||||
|
||||
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX +
|
||||
WHITELIST_GROUP, openSocketPolicyFile.toString());
|
||||
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX +
|
||||
NORMAL_GROUP, classLoaderPolicyFile.toString());
|
||||
|
||||
String[] inputCommand = {"$JAVA_HOME/bin/java jar MyJob.jar"};
|
||||
List<String> commands = Arrays.asList(inputCommand);
|
||||
|
||||
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
|
||||
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
|
||||
|
||||
runtime.prepareContainer(runtimeContextBuilder.build());
|
||||
|
||||
//pull generated policy from cmd
|
||||
Matcher policyMatches = Pattern.compile(POLICY_APPEND_FLAG + "=?([^ ]+)")
|
||||
.matcher(commands.get(0));
|
||||
policyMatches.find();
|
||||
String generatedPolicy = policyMatches.group(1);
|
||||
|
||||
//Test that generated policy file has included both policies
|
||||
Assert.assertTrue(
|
||||
Files.readAllLines(Paths.get(generatedPolicy)).contains(
|
||||
classLoaderPermString.toString().split("\n")[1]));
|
||||
Assert.assertTrue(
|
||||
Files.readAllLines(Paths.get(generatedPolicy)).contains(
|
||||
socketPermString.toString().split("\n")[1]));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGrant() throws Exception {
|
||||
FilePermission grantPermission =
|
||||
@ -235,7 +308,6 @@ public void testPermissiveMode() throws ContainerExecutionException {
|
||||
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
|
||||
.appendSecurityFlags(commands, env, policyFilePath,
|
||||
JavaSandboxLinuxContainerRuntime.SandboxMode.permissive);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -247,6 +319,9 @@ public void testDisabledSandboxWithWhitelist()
|
||||
};
|
||||
List<String> commands = Arrays.asList(inputCommand);
|
||||
|
||||
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
|
||||
WHITELIST_GROUP);
|
||||
|
||||
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
|
||||
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
|
||||
runtime.prepareContainer(runtimeContextBuilder.build());
|
||||
@ -254,7 +329,6 @@ public void testDisabledSandboxWithWhitelist()
|
||||
Assert.assertTrue("Command should not be modified when user is " +
|
||||
"member of whitelisted group",
|
||||
inputCommand[0].equals(commands.get(0)));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -265,6 +339,9 @@ public void testEnabledSandboxWithWhitelist()
|
||||
};
|
||||
List<String> commands = Arrays.asList(inputCommand);
|
||||
|
||||
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
|
||||
WHITELIST_GROUP);
|
||||
|
||||
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
|
||||
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
|
||||
runtime.prepareContainer(runtimeContextBuilder.build());
|
||||
@ -283,6 +360,9 @@ public void testDeniedWhitelistGroup() throws ContainerExecutionException {
|
||||
};
|
||||
List<String> commands = Arrays.asList(inputCommand);
|
||||
|
||||
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
|
||||
WHITELIST_GROUP);
|
||||
|
||||
runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER);
|
||||
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
|
||||
runtime.prepareContainer(runtimeContextBuilder.build());
|
||||
@ -373,6 +453,6 @@ public void testAppendSecurityFlags() throws ContainerExecutionException {
|
||||
|
||||
@After
|
||||
public void cleanup(){
|
||||
System.setSecurityManager(null);
|
||||
System.setProperties(baseProps);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user