big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2194. Fix bug causing CGroups functionality to fail on RHEL7. Contributed by Wei Yan.

Browse Source
This commit is contained in:
Varun Vasudev 2015-07-07 16:59:29 +05:30
parent 99c8c5839b
commit c40bdb56a7
10 changed files with 44 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
View File

@ -872,6 +872,10 @@ public static String join(CharSequence separator, Iterable<?> strings) {
return sb.toString();
}
public static String join(char separator, Iterable<?> strings) {
return join(separator + "", strings);
}
/**
* Concatenates strings, using a separator.
*
@ -894,6 +898,10 @@ public static String join(CharSequence separator, String[] strings) {
return sb.toString();
}
public static String join(char separator, String[] strings) {
return join(separator + "", strings);
}
/**
* Convert SOME_STUFF to SomeStuff
*

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
View File

@ -278,8 +278,12 @@ public void testJoin() {
s.add("c");
assertEquals("", StringUtils.join(":", s.subList(0, 0)));
assertEquals("a", StringUtils.join(":", s.subList(0, 1)));
assertEquals("", StringUtils.join(':', s.subList(0, 0)));
assertEquals("a", StringUtils.join(':', s.subList(0, 1)));
assertEquals("a:b", StringUtils.join(":", s.subList(0, 2)));
assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3)));
assertEquals("a:b", StringUtils.join(':', s.subList(0, 2)));
assertEquals("a:b:c", StringUtils.join(':', s.subList(0, 3)));
}
@Test (timeout = 30000)

3
hadoop-yarn-project/CHANGES.txt
View File

@ -595,6 +595,9 @@ Release 2.8.0 - UNRELEASED
YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong
prefix for auth options. (Bibin A Chundatt via devaraj)
YARN-2194. Fix bug causing CGroups functionality to fail on RHEL7.
(Wei Yan via vvasudev)
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
View File

@ -241,8 +241,10 @@ public void startLocalizer(LocalizerStartContext ctx)
Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()),
appId,
nmPrivateContainerTokensPath.toUri().getPath().toString(),
StringUtils.join(",", localDirs),
StringUtils.join(",", logDirs)));
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
localDirs),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
logDirs)));
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
@ -363,8 +365,10 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
nmPrivateContainerScriptPath.toUri().getPath().toString(),
nmPrivateTokensPath.toUri().getPath().toString(),
pidFilePath.toString(),
StringUtils.join(",", localDirs),
StringUtils.join(",", logDirs),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
localDirs),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
logDirs),
resourcesOptions));
if (tcCommandFile != null) {

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
View File

@ -36,6 +36,7 @@
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class PrivilegedOperation {
public final static char LINUX_FILE_PATH_SEPARATOR = '%';
public enum OperationType {
CHECK_SETUP("--checksetup"),

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
View File

@ -234,7 +234,7 @@ public String executePrivilegedOperation(PrivilegedOperation operation,
if (noneArgsOnly == false) {
//We have already appended at least one tasks file.
finalOpArg.append(",");
finalOpArg.append(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR);
finalOpArg.append(tasksFile);
} else {
finalOpArg.append(tasksFile);

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
View File

@ -49,6 +49,7 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.SystemClock;
@ -409,10 +410,11 @@ public String getResourcesOption(ContainerId containerId) {
if (isCpuWeightEnabled()) {
sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/tasks");
sb.append(",");
sb.append(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR);
}
if (sb.charAt(sb.length() - 1) == ',') {
if (sb.charAt(sb.length() - 1) ==
PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR) {
sb.deleteCharAt(sb.length() - 1);
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
View File

@ -283,7 +283,7 @@ char * get_value(const char* key) {
/**
* Function to return an array of values for a key.
* Value delimiter is assumed to be a comma.
* Value delimiter is assumed to be a '%'.
*/
char ** get_values(const char * key) {
char *value = get_value(key);
@ -291,7 +291,7 @@ char ** get_values(const char * key) {
}
/**
* Extracts array of values from the comma separated list of values.
* Extracts array of values from the '%' separated list of values.
*/
char ** extract_values(char *value) {
char ** toPass = NULL;
@ -303,14 +303,14 @@ char ** extract_values(char *value) {
//first allocate any array of 10
if(value != NULL) {
toPass = (char **) malloc(sizeof(char *) * toPassSize);
tempTok = strtok_r((char *)value, ",", &tempstr);
tempTok = strtok_r((char *)value, "%", &tempstr);
while (tempTok != NULL) {
toPass[size++] = tempTok;
if(size == toPassSize) {
toPassSize += MAX_SIZE;
toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
}
tempTok = strtok_r(NULL, ",", &tempstr);
tempTok = strtok_r(NULL, "%", &tempstr);
}
}
if (toPass != NULL) {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
View File

@ -30,8 +30,8 @@
#define TEST_ROOT "/tmp/test-container-executor"
#define DONT_TOUCH_FILE "dont-touch-me"
#define NM_LOCAL_DIRS TEST_ROOT "/local-1," TEST_ROOT "/local-2," \
TEST_ROOT "/local-3," TEST_ROOT "/local-4," TEST_ROOT "/local-5"
#define NM_LOCAL_DIRS TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \
TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5"
#define NM_LOG_DIRS TEST_ROOT "/logs/userlogs"
#define ARRAY_SIZE 1000

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
View File

@ -49,6 +49,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
@ -148,8 +149,10 @@ public void testContainerLaunch() throws IOException {
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, appId, containerId,
workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
StringUtils.join(",", dirsHandler.getLocalDirs()),
StringUtils.join(",", dirsHandler.getLogDirs()), "cgroups=none"),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
dirsHandler.getLocalDirs()),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
dirsHandler.getLogDirs()), "cgroups=none"),
readMockParams());
}
@ -312,8 +315,10 @@ public Object answer(InvocationOnMock invocationOnMock)
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, appId, containerId,
workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
StringUtils.join(",", dirsHandler.getLocalDirs()),
StringUtils.join(",", dirsHandler.getLogDirs()),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
dirsHandler.getLocalDirs()),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
dirsHandler.getLogDirs()),
"cgroups=none"), readMockParams());
}