big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4727. Unable to override the $HADOOP_CONF_DIR env variable for container. Contributed by Jason Lowe.

Browse Source
This commit is contained in:
Eric Payne 2017-09-13 14:41:55 -05:00
parent a3c44195be
commit 3860be7961
2 changed files with 24 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
View File

@ -1148,10 +1148,7 @@ public void sanitizeEnv(Map<String, String> environment, Path pwd,
environment.put(Environment.PWD.name(), pwd.toString());
putEnvIfNotNull(environment,
Environment.HADOOP_CONF_DIR.name(),
System.getenv(Environment.HADOOP_CONF_DIR.name())
);
putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name());
if (!Shell.WINDOWS) {
environment.put("JVM_PID", "$$");

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
View File

@ -46,6 +46,7 @@
import java.util.jar.JarFile;
import java.util.jar.Manifest;
import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
@ -54,6 +55,7 @@
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.StringUtils;
@ -724,11 +726,15 @@ public void testContainerEnvVariables() throws Exception {
userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
final String userConfDir = "user_set_HADOOP_CONF_DIR";
userSetEnv.put(Environment.HADOOP_CONF_DIR.name(), userConfDir);
containerLaunchContext.setEnvironment(userSetEnv);
File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
File processStartFile =
new File(tmpDir, "env_vars.tmp").getAbsoluteFile();
final File processFinalFile =
new File(tmpDir, "env_vars.txt").getAbsoluteFile();
if (Shell.WINDOWS) {
fileWriter.println("@echo " + Environment.CONTAINER_ID.$() + "> "
@ -749,6 +755,8 @@ public void testContainerEnvVariables() throws Exception {
+ processStartFile);
fileWriter.println("@echo " + Environment.HOME.$() + ">> "
+ processStartFile);
fileWriter.println("@echo " + Environment.HADOOP_CONF_DIR.$() + ">> "
+ processStartFile);
for (String serviceName : containerManager.getAuxServiceMetaData()
.keySet()) {
fileWriter.println("@echo %" + AuxiliaryServiceHelper.NM_AUX_SERVICE
@ -756,6 +764,8 @@ public void testContainerEnvVariables() throws Exception {
+ processStartFile);
}
fileWriter.println("@echo " + cId + ">> " + processStartFile);
fileWriter.println("@move /Y " + processStartFile + " "
+ processFinalFile);
fileWriter.println("@ping -n 100 127.0.0.1 >nul");
} else {
fileWriter.write("\numask 0"); // So that start file is readable by the test
@ -777,6 +787,8 @@ public void testContainerEnvVariables() throws Exception {
+ processStartFile);
fileWriter.write("\necho $" + Environment.HOME.name() + " >> "
+ processStartFile);
fileWriter.write("\necho $" + Environment.HADOOP_CONF_DIR.name() + " >> "
+ processStartFile);
for (String serviceName : containerManager.getAuxServiceMetaData()
.keySet()) {
fileWriter.write("\necho $" + AuxiliaryServiceHelper.NM_AUX_SERVICE
@ -784,6 +796,7 @@ public void testContainerEnvVariables() throws Exception {
+ processStartFile);
}
fileWriter.write("\necho $$ >> " + processStartFile);
fileWriter.write("\nmv " + processStartFile + " " + processFinalFile);
fileWriter.write("\nexec sleep 100");
}
fileWriter.close();
@ -817,13 +830,12 @@ public void testContainerEnvVariables() throws Exception {
StartContainersRequest.newInstance(list);
containerManager.startContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.exists() && timeoutSecs++ < 20) {
Thread.sleep(1000);
LOG.info("Waiting for process start-file to be created");
}
Assert.assertTrue("ProcessStartFile doesn't exist!",
processStartFile.exists());
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
return processFinalFile.exists();
}
}, 10, 20000);
// Now verify the contents of the file
List<String> localDirs = dirsHandler.getLocalDirs();
@ -843,7 +855,7 @@ public void testContainerEnvVariables() throws Exception {
containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
}
BufferedReader reader =
new BufferedReader(new FileReader(processStartFile));
new BufferedReader(new FileReader(processFinalFile));
Assert.assertEquals(cId.toString(), reader.readLine());
Assert.assertEquals(context.getNodeId().getHost(), reader.readLine());
Assert.assertEquals(String.valueOf(context.getNodeId().getPort()),
@ -866,7 +878,7 @@ public void testContainerEnvVariables() throws Exception {
YarnConfiguration.NM_USER_HOME_DIR,
YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
reader.readLine());
Assert.assertEquals(userConfDir, reader.readLine());
for (String serviceName : containerManager.getAuxServiceMetaData().keySet()) {
Assert.assertEquals(
containerManager.getAuxServiceMetaData().get(serviceName),
@ -905,6 +917,8 @@ public void testContainerEnvVariables() throws Exception {
YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
containerLaunchContext.getEnvironment()
.get(Environment.HOME.name()));
Assert.assertEquals(userConfDir, containerLaunchContext.getEnvironment()
.get(Environment.HADOOP_CONF_DIR.name()));
// Get the pid of the process
String pid = reader.readLine().trim();