big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9092. Create an object for cgroups mount enable and cgroups mount path as they belong together. Contributed by Gergely Pollak

Browse Source
This commit is contained in:
Szilard Nemeth 2019-08-09 10:18:34 +02:00
parent 742e30b473
commit e0c21c6da9
3 changed files with 91 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
View File

@ -66,8 +66,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
private final String mtabFile;
private final String cGroupPrefix;
private final boolean enableCGroupMount;
private final String cGroupMountPath;
private final CGroupsMountConfig cGroupsMountConfig;
private final long deleteCGroupTimeout;
private final long deleteCGroupDelay;
private Map<CGroupController, String> controllerPaths;
@ -91,10 +90,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
this.cGroupPrefix = conf.get(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "/hadoop-yarn")
.replaceAll("^/+", "").replaceAll("/+$", "");
this.enableCGroupMount = conf.getBoolean(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT, false);
this.cGroupMountPath = conf.get(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, null);
this.cGroupsMountConfig = new CGroupsMountConfig(conf);
this.deleteCGroupTimeout = conf.getLong(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT,
YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT) +
@ -150,9 +146,9 @@ private void initializeControllerPaths() throws ResourceHandlerException {
Map<String, Set<String>> newMtab = null;
Map<CGroupController, String> cPaths;
try {
if (this.cGroupMountPath != null && !this.enableCGroupMount) {
if (this.cGroupsMountConfig.mountDisabledButMountPathDefined()) {
newMtab = ResourceHandlerModule.
parseConfiguredCGroupPath(this.cGroupMountPath);
parseConfiguredCGroupPath(this.cGroupsMountConfig.getMountPath());
}
if (newMtab == null) {
@ -282,14 +278,10 @@ static String findControllerInMtab(String controller,
private void mountCGroupController(CGroupController controller)
throws ResourceHandlerException {
if (cGroupMountPath == null) {
throw new ResourceHandlerException(
String.format("Cgroups mount path not specified in %s.",
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH));
}
String existingMountPath = getControllerPath(controller);
String requestedMountPath =
new File(cGroupMountPath, controller.getName()).getAbsolutePath();
new File(cGroupsMountConfig.getMountPath(),
controller.getName()).getAbsolutePath();
if (existingMountPath == null ||
!requestedMountPath.equals(existingMountPath)) {
@ -367,7 +359,8 @@ public String getPathForCGroupParam(CGroupController controller,
@Override
public void initializeCGroupController(CGroupController controller) throws
ResourceHandlerException {
if (enableCGroupMount) {
if (this.cGroupsMountConfig.isMountEnabled() &&
cGroupsMountConfig.ensureMountPathIsDefined()) {
// We have a controller that needs to be mounted
mountCGroupController(controller);
}
@ -611,7 +604,7 @@ public String getCGroupParam(CGroupController controller, String cGroupId,
@Override
public String getCGroupMountPath() {
return cGroupMountPath;
return this.cGroupsMountConfig.getMountPath();
}
@Override
@ -619,8 +612,7 @@ public String toString() {
return CGroupsHandlerImpl.class.getName() + "{" +
"mtabFile='" + mtabFile + '\'' +
", cGroupPrefix='" + cGroupPrefix + '\'' +
", enableCGroupMount=" + enableCGroupMount +
", cGroupMountPath='" + cGroupMountPath + '\'' +
", cGroupsMountConfig=" + cGroupsMountConfig +
", deleteCGroupTimeout=" + deleteCGroupTimeout +
", deleteCGroupDelay=" + deleteCGroupDelay +
'}';

72
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMountConfig.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
/**
* Stores config related to cgroups.
*/
public class CGroupsMountConfig {
private final boolean enableMount;
private final String mountPath;
public CGroupsMountConfig(Configuration conf) {
this.enableMount = conf.getBoolean(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT, false);
this.mountPath = conf.get(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, null);
}
public boolean ensureMountPathIsDefined() throws ResourceHandlerException {
if (mountPath == null) {
throw new ResourceHandlerException(
String.format("Cgroups mount path not specified in %s.",
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH));
}
return true;
}
public boolean isMountPathDefined() {
return mountPath != null;
}
public boolean isMountEnabled() {
return enableMount;
}
public boolean mountDisabledButMountPathDefined() {
return !enableMount && mountPath != null;
}
public boolean mountEnabledAndMountPathDefined() {
return enableMount && mountPath != null;
}
public String getMountPath() {
return mountPath;
}
@Override
public String toString() {
return "CGroupsMountConfig{" +
"enableMount=" + enableMount +
", mountPath='" + mountPath + '\'' +
'}';
}
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
View File

@ -37,6 +37,8 @@
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsMountConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -73,8 +75,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
private Configuration conf;
private String cgroupPrefix;
private boolean cgroupMount;
private String cgroupMountPath;
private CGroupsMountConfig cGroupsMountConfig;
private boolean cpuWeightEnabled = true;
private boolean strictResourceUsageMode = false;
@ -115,11 +116,7 @@ void initConfig() throws IOException {
this.cgroupPrefix = conf.get(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "/hadoop-yarn");
this.cgroupMount = conf.getBoolean(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT, false);
this.cgroupMountPath = conf.get(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, null);
this.cGroupsMountConfig = new CGroupsMountConfig(conf);
this.deleteCgroupTimeout = conf.getLong(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT,
YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT);
@ -156,10 +153,10 @@ void init(LinuxContainerExecutor lce, ResourceCalculatorPlugin plugin)
initConfig();
// mount cgroups if requested
if (cgroupMount && cgroupMountPath != null) {
if (cGroupsMountConfig.mountEnabledAndMountPathDefined()) {
ArrayList<String> cgroupKVs = new ArrayList<String>();
cgroupKVs.add(CONTROLLER_CPU + "=" + cgroupMountPath + "/" +
CONTROLLER_CPU);
cgroupKVs.add(CONTROLLER_CPU + "=" +
cGroupsMountConfig.getMountPath() + "/" + CONTROLLER_CPU);
lce.mountCgroups(cgroupKVs, cgroupPrefix);
}
@ -456,9 +453,9 @@ private void initializeControllerPaths() throws IOException {
String controllerPath;
Map<String, Set<String>> parsedMtab = null;
if (this.cgroupMountPath != null && !this.cgroupMount) {
if (this.cGroupsMountConfig.mountDisabledButMountPathDefined()) {
parsedMtab = ResourceHandlerModule.
parseConfiguredCGroupPath(this.cgroupMountPath);
parseConfiguredCGroupPath(this.cGroupsMountConfig.getMountPath());
}
if (parsedMtab == null) {