big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native executor returns an error. Contributed by Eric Payne

Browse Source
This commit is contained in:
Jason Lowe 2015-01-26 15:40:21 +00:00
parent 2b0fa20f69
commit 902c6ea7e4
3 changed files with 58 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -409,6 +409,9 @@ Release 2.7.0 - UNRELEASED
YARN-3082. Non thread safe access to systemCredentials in NodeHeartbeatResponse YARN-3082. Non thread safe access to systemCredentials in NodeHeartbeatResponse
processing. (Anubhav Dhoot via ozawa) processing. (Anubhav Dhoot via ozawa)
YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native
executor returns an error (Eric Payne via jlowe)
Release 2.6.0 - 2014-11-18 Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
View File

@ -392,18 +392,23 @@ public void deleteAsUser(String user, Path dir, Path... baseDirs) {
verifyUsernamePattern(user); verifyUsernamePattern(user);
String runAsUser = getRunAsUser(user); String runAsUser = getRunAsUser(user);
String dirString = dir == null ? "" : dir.toUri().getPath();
List<String> command = new ArrayList<String>( List<String> command = new ArrayList<String>(
Arrays.asList(containerExecutorExe, Arrays.asList(containerExecutorExe,
runAsUser, runAsUser,
user, user,
Integer.toString(Commands.DELETE_AS_USER.getValue()), Integer.toString(Commands.DELETE_AS_USER.getValue()),
dir == null ? "" : dir.toUri().getPath())); dirString));
List<String> pathsToDelete = new ArrayList<String>();
if (baseDirs == null || baseDirs.length == 0) { if (baseDirs == null || baseDirs.length == 0) {
LOG.info("Deleting absolute path : " + dir); LOG.info("Deleting absolute path : " + dir);
pathsToDelete.add(dirString);
} else { } else {
for (Path baseDir : baseDirs) { for (Path baseDir : baseDirs) {
Path del = dir == null ? baseDir : new Path(baseDir, dir); Path del = dir == null ? baseDir : new Path(baseDir, dir);
LOG.info("Deleting path : " + del); LOG.info("Deleting path : " + del);
pathsToDelete.add(del.toString());
command.add(baseDir.toUri().getPath()); command.add(baseDir.toUri().getPath());
} }
} }
@ -419,7 +424,7 @@ public void deleteAsUser(String user, Path dir, Path... baseDirs) {
} }
} catch (IOException e) { } catch (IOException e) {
int exitCode = shExec.getExitCode(); int exitCode = shExec.getExitCode();
LOG.error("DeleteAsUser for " + dir.toUri().getPath() LOG.error("DeleteAsUser for " + StringUtils.join(" ", pathsToDelete)
+ " returned with exit code: " + exitCode, e); + " returned with exit code: " + exitCode, e);
LOG.error("Output from LinuxContainerExecutor's deleteAsUser follows:"); LOG.error("Output from LinuxContainerExecutor's deleteAsUser follows:");
logOutput(shExec.getOutput()); logOutput(shExec.getOutput());

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
View File

@ -319,10 +319,57 @@ public void testDeleteAsUser() throws IOException {
String cmd = String.valueOf( String cmd = String.valueOf(
LinuxContainerExecutor.Commands.DELETE_AS_USER.getValue()); LinuxContainerExecutor.Commands.DELETE_AS_USER.getValue());
Path dir = new Path("/tmp/testdir"); Path dir = new Path("/tmp/testdir");
Path testFile = new Path("testfile");
Path baseDir0 = new Path("/grid/0/BaseDir");
Path baseDir1 = new Path("/grid/1/BaseDir");
mockExec.deleteAsUser(appSubmitter, dir); mockExec.deleteAsUser(appSubmitter, dir);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, "/tmp/testdir"), appSubmitter, cmd, "/tmp/testdir"),
readMockParams()); readMockParams());
mockExec.deleteAsUser(appSubmitter, null);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, ""),
readMockParams());
mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
readMockParams());
mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
readMockParams());
File f = new File("./src/test/resources/mock-container-executer-with-error");
if (!FileUtil.canExecute(f)) {
FileUtil.setExecutable(f, true);
}
String executorPath = f.getAbsolutePath();
Configuration conf = new Configuration();
conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
mockExec.setConf(conf);
mockExec.deleteAsUser(appSubmitter, dir);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, "/tmp/testdir"),
readMockParams());
mockExec.deleteAsUser(appSubmitter, null);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, ""),
readMockParams());
mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
readMockParams());
mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
readMockParams());
} }
} }