big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5383. Fix findbugs and checkstyle issues in ContainerExecutor. (Vrushali C via Varun Saxena)

Browse Source
This commit is contained in:
Varun Saxena 2016-07-15 23:17:04 +05:30
parent 7e5355c14e
commit a72cb3825a
1 changed files with 27 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
View File

@ -305,7 +305,7 @@ public void writeLaunchEnv(OutputStream out,
List<String> command, Path logDir, String outFilename) List<String> command, Path logDir, String outFilename)
throws IOException { throws IOException {
ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create(); ContainerLaunch.ShellScriptBuilder.create();
Set<String> whitelist = new HashSet<>(); Set<String> whitelist = new HashSet<>();
whitelist.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME); whitelist.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME);
@ -316,7 +316,7 @@ public void writeLaunchEnv(OutputStream out,
whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name()); whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name());
if (environment != null) { if (environment != null) {
for (Map.Entry<String,String> env : environment.entrySet()) { for (Map.Entry<String, String> env : environment.entrySet()) {
if (!whitelist.contains(env.getKey())) { if (!whitelist.contains(env.getKey())) {
sb.env(env.getKey(), env.getValue()); sb.env(env.getKey(), env.getValue());
} else { } else {
@ -326,19 +326,20 @@ public void writeLaunchEnv(OutputStream out,
} }
if (resources != null) { if (resources != null) {
for (Path path: resources.keySet()) { for (Map.Entry<Path, List<String>> resourceEntry :
for (String linkName: resources.get(path)) { resources.entrySet()) {
for (String linkName : resourceEntry.getValue()) {
if (new Path(linkName).getName().equals(WILDCARD)) { if (new Path(linkName).getName().equals(WILDCARD)) {
// If this is a wildcarded path, link to everything in the // If this is a wildcarded path, link to everything in the
// directory from the working directory // directory from the working directory
File directory = new File(path.toString()); File directory = new File(resourceEntry.getKey().toString());
for (File wildLink : directory.listFiles()) { for (File wildLink : directory.listFiles()) {
sb.symlink(new Path(wildLink.toString()), sb.symlink(new Path(wildLink.toString()),
new Path(wildLink.getName())); new Path(wildLink.getName()));
} }
} else { } else {
sb.symlink(path, new Path(linkName)); sb.symlink(resourceEntry.getKey(), new Path(linkName));
} }
} }
} }
@ -467,12 +468,12 @@ protected Path getPidFilePath(ContainerId containerId) {
* @param groupId the job owner's GID * @param groupId the job owner's GID
* @param userName the job owner's username * @param userName the job owner's username
* @param pidFile the path to the container's PID file * @param pidFile the path to the container's PID file
* @param conf the configuration * @param config the configuration
* @return the command line to execute * @return the command line to execute
*/ */
protected String[] getRunCommand(String command, String groupId, protected String[] getRunCommand(String command, String groupId,
String userName, Path pidFile, Configuration conf) { String userName, Path pidFile, Configuration config) {
return getRunCommand(command, groupId, userName, pidFile, conf, null); return getRunCommand(command, groupId, userName, pidFile, config, null);
} }
/** /**
@ -488,18 +489,18 @@ protected String[] getRunCommand(String command, String groupId,
* systems it is ignored. * systems it is ignored.
* @param pidFile the path to the container's PID file on Windows. On other * @param pidFile the path to the container's PID file on Windows. On other
* operating systems it is ignored. * operating systems it is ignored.
* @param conf the configuration * @param config the configuration
* @param resource on Windows this parameter controls memory and CPU limits. * @param resource on Windows this parameter controls memory and CPU limits.
* If null, no limits are set. On other operating systems it is ignored. * If null, no limits are set. On other operating systems it is ignored.
* @return the command line to execute * @return the command line to execute
*/ */
protected String[] getRunCommand(String command, String groupId, protected String[] getRunCommand(String command, String groupId,
String userName, Path pidFile, Configuration conf, Resource resource) { String userName, Path pidFile, Configuration config, Resource resource) {
if (Shell.WINDOWS) { if (Shell.WINDOWS) {
return getRunCommandForWindows(command, groupId, userName, pidFile, return getRunCommandForWindows(command, groupId, userName, pidFile,
conf, resource); config, resource);
} else { } else {
return getRunCommandForOther(command, conf); return getRunCommandForOther(command, config);
} }
} }
@ -513,30 +514,31 @@ protected String[] getRunCommand(String command, String groupId,
* @param groupId the job owner's GID * @param groupId the job owner's GID
* @param userName the job owner's username * @param userName the job owner's username
* @param pidFile the path to the container's PID file * @param pidFile the path to the container's PID file
* @param conf the configuration * @param config the configuration
* @param resource this parameter controls memory and CPU limits. * @param resource this parameter controls memory and CPU limits.
* If null, no limits are set. * If null, no limits are set.
* @return the command line to execute * @return the command line to execute
*/ */
protected String[] getRunCommandForWindows(String command, String groupId, protected String[] getRunCommandForWindows(String command, String groupId,
String userName, Path pidFile, Configuration conf, Resource resource) { String userName, Path pidFile, Configuration config, Resource resource) {
int cpuRate = -1; int cpuRate = -1;
int memory = -1; int memory = -1;
if (resource != null) { if (resource != null) {
if (conf.getBoolean( if (config.getBoolean(
YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED, YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) { YarnConfiguration.
memory = (int)resource.getMemorySize(); DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
memory = (int) resource.getMemorySize();
} }
if (conf.getBoolean( if (config.getBoolean(
YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED, YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) { YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
int containerVCores = resource.getVirtualCores(); int containerVCores = resource.getVirtualCores();
int nodeVCores = NodeManagerHardwareUtils.getVCores(conf); int nodeVCores = NodeManagerHardwareUtils.getVCores(config);
int nodeCpuPercentage = int nodeCpuPercentage =
NodeManagerHardwareUtils.getNodeCpuPercentage(conf); NodeManagerHardwareUtils.getNodeCpuPercentage(config);
float containerCpuPercentage = float containerCpuPercentage =
(float)(nodeCpuPercentage * containerVCores) / nodeVCores; (float)(nodeCpuPercentage * containerVCores) / nodeVCores;
@ -564,20 +566,20 @@ protected String[] getRunCommandForWindows(String command, String groupId,
* Return a command line to execute the given command in the OS shell. * Return a command line to execute the given command in the OS shell.
* *
* @param command the command to execute * @param command the command to execute
* @param conf the configuration * @param config the configuration
* @return the command line to execute * @return the command line to execute
*/ */
protected String[] getRunCommandForOther(String command, protected String[] getRunCommandForOther(String command,
Configuration conf) { Configuration config) {
List<String> retCommand = new ArrayList<>(); List<String> retCommand = new ArrayList<>();
boolean containerSchedPriorityIsSet = false; boolean containerSchedPriorityIsSet = false;
int containerSchedPriorityAdjustment = int containerSchedPriorityAdjustment =
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY; YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;
if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != if (config.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) !=
null) { null) {
containerSchedPriorityIsSet = true; containerSchedPriorityIsSet = true;
containerSchedPriorityAdjustment = conf containerSchedPriorityAdjustment = config
.getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY,
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY); YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
} }