big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9087. Improve logging for initialization of Resource plugins. (Contributed by Szilard Nemeth)

Browse Source
This commit is contained in:
Haibo Chen 2018-12-10 11:12:54 -08:00
parent 64411a6ff7
commit ac578c0e82
18 changed files with 155 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
View File

@ -315,11 +315,12 @@ public void init(Context context) throws IOException {
resourceHandlerChain = ResourceHandlerModule
.getConfiguredResourceHandlerChain(conf, nmContext);
if (LOG.isDebugEnabled()) {
LOG.debug("Resource handler chain enabled = " + (resourceHandlerChain
!= null));
final boolean enabled = resourceHandlerChain != null;
LOG.debug("Resource handler chain enabled = " + enabled);
}
if (resourceHandlerChain != null) {
LOG.debug("Bootstrapping resource handler chain");
LOG.debug("Bootstrapping resource handler chain: " +
resourceHandlerChain);
resourceHandlerChain.bootstrap(conf);
}
} catch (ResourceHandlerException e) {

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
View File

@ -173,4 +173,9 @@ public List<PrivilegedOperation> postComplete(ContainerId containerId)
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return CGroupsBlkioResourceHandlerImpl.class.getName();
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
View File

@ -256,4 +256,9 @@ public List<PrivilegedOperation> postComplete(ContainerId containerId)
throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return CGroupsCpuResourceHandlerImpl.class.getName();
}
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
View File

@ -64,7 +64,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
private static final String MTAB_FILE = "/proc/mounts";
private static final String CGROUPS_FSTYPE = "cgroup";
private String mtabFile;
private final String mtabFile;
private final String cGroupPrefix;
private final boolean enableCGroupMount;
private final String cGroupMountPath;
@ -622,4 +622,16 @@ public String getCGroupParam(CGroupController controller, String cGroupId,
public String getCGroupMountPath() {
return cGroupMountPath;
}
@Override
public String toString() {
return CGroupsHandlerImpl.class.getName() + "{" +
"mtabFile='" + mtabFile + '\'' +
", cGroupPrefix='" + cGroupPrefix + '\'' +
", enableCGroupMount=" + enableCGroupMount +
", cGroupMountPath='" + cGroupMountPath + '\'' +
", deleteCGroupTimeout=" + deleteCGroupTimeout +
", deleteCGroupDelay=" + deleteCGroupDelay +
'}';
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
View File

@ -171,4 +171,9 @@ public List<PrivilegedOperation> postComplete(ContainerId containerId)
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return CGroupsMemoryResourceHandlerImpl.class.getName();
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkPacketTaggingHandlerImpl.java
View File

@ -166,4 +166,9 @@ public NetworkTagMappingManager createNetworkTagMappingManager(
Configuration conf) {
return NetworkTagMappingManagerFactory.getManager(conf);
}
@Override
public String toString() {
return NetworkPacketTaggingHandlerImpl.class.getName();
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
View File

@ -159,4 +159,10 @@ public List<ResourceHandler> getResourceHandlerList() {
return Collections.unmodifiableList(resourceHandlers);
}
@Override
public String toString() {
return ResourceHandlerChain.class.getName() + "{" +
"resourceHandlers=" + resourceHandlers +
'}';
}
}

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
View File

@ -84,6 +84,9 @@ private static CGroupsHandler getInitializedCGroupsHandler(Configuration conf)
if (cGroupsHandler == null) {
cGroupsHandler = new CGroupsHandlerImpl(conf,
PrivilegedOperationExecutor.getInstance(conf));
if (LOG.isDebugEnabled()) {
LOG.debug("Value of CGroupsHandler is: " + cGroupsHandler);
}
}
}
}
@ -306,16 +309,32 @@ private static void addHandlersFromConfiguredResourcePlugins(
List<ResourceHandler> handlerList, Configuration conf,
Context nmContext) throws ResourceHandlerException {
ResourcePluginManager pluginManager = nmContext.getResourcePluginManager();
if (pluginManager != null) {
Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
if (pluginMap != null) {
for (ResourcePlugin plugin : pluginMap.values()) {
addHandlerIfNotNull(handlerList, plugin
.createResourceHandler(nmContext,
getInitializedCGroupsHandler(conf),
PrivilegedOperationExecutor.getInstance(conf)));
}
if (pluginManager == null) {
LOG.warn("Plugin manager was null while trying to add " +
"ResourceHandlers from configuration!");
return;
}
Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
if (pluginMap == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("List of plugins of ResourcePluginManager was empty " +
"while trying to add ResourceHandlers from configuration!");
}
return;
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("List of plugins of ResourcePluginManager: " +
pluginManager.getNameToPlugins());
}
}
for (ResourcePlugin plugin : pluginMap.values()) {
addHandlerIfNotNull(handlerList,
plugin.createResourceHandler(nmContext,
getInitializedCGroupsHandler(conf),
PrivilegedOperationExecutor.getInstance(conf)));
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
View File

@ -283,4 +283,9 @@ public List<PrivilegedOperation> teardown()
return null;
}
@Override
public String toString() {
return TrafficControlBandwidthHandlerImpl.class.getName();
}
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
View File

@ -55,11 +55,11 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
private final String REQUEST_FPGA_IP_ID_KEY = "REQUESTED_FPGA_IP_ID";
private AbstractFpgaVendorPlugin vendorPlugin;
private final AbstractFpgaVendorPlugin vendorPlugin;
private FpgaResourceAllocator allocator;
private final FpgaResourceAllocator allocator;
private CGroupsHandler cGroupsHandler;
private final CGroupsHandler cGroupsHandler;
public static final String EXCLUDED_FPGAS_CLI_OPTION = "--excluded_fpgas";
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
@ -223,4 +223,12 @@ public List<PrivilegedOperation> postComplete(ContainerId containerId) throws Re
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return FpgaResourceHandlerImpl.class.getName() + "{" +
"vendorPlugin=" + vendorPlugin +
", allocator=" + allocator +
'}';
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
View File

@ -304,4 +304,9 @@ public synchronized List<AssignedGpuDevice> getAssignedGpusCopy() {
}
return assigns;
}
@Override
public String toString() {
return GpuResourceAllocator.class.getName();
}
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
View File

@ -48,10 +48,10 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
private Context nmContext;
private GpuResourceAllocator gpuAllocator;
private CGroupsHandler cGroupsHandler;
private PrivilegedOperationExecutor privilegedOperationExecutor;
private final Context nmContext;
private final GpuResourceAllocator gpuAllocator;
private final CGroupsHandler cGroupsHandler;
private final PrivilegedOperationExecutor privilegedOperationExecutor;
public GpuResourceHandlerImpl(Context nmContext,
CGroupsHandler cGroupsHandler,
@ -185,4 +185,11 @@ public synchronized List<PrivilegedOperation> postComplete(
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return GpuResourceHandlerImpl.class.getName() + "{" +
"gpuAllocator=" + gpuAllocator +
'}';
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceHandlerImpl.java
View File

@ -41,8 +41,8 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
private static final Log LOG = LogFactory
.getLog(NumaResourceHandlerImpl.class);
private NumaResourceAllocator numaResourceAllocator;
private String numaCtlCmd;
private final NumaResourceAllocator numaResourceAllocator;
private final String numaCtlCmd;
public NumaResourceHandlerImpl(Configuration conf, Context nmContext) {
LOG.info("NUMA resources allocation is enabled, initializing NUMA resources"
@ -111,4 +111,12 @@ public List<PrivilegedOperation> postComplete(ContainerId containerId)
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return NumaResourceHandlerImpl.class.getName() + "{" +
"numaResourceAllocator=" + numaResourceAllocator +
", numaCtlCmd='" + numaCtlCmd + '\'' +
'}';
}
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
View File

@ -39,6 +39,7 @@
import org.slf4j.LoggerFactory;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@ -67,6 +68,12 @@ public synchronized void initialize(Context context)
Map<String, ResourcePlugin> pluginMap = new HashMap<>();
String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
if (plugins == null || plugins.length == 0) {
LOG.info("No Resource plugins found from configuration!");
}
LOG.info("Found Resource plugins from configuration: "
+ Arrays.toString(plugins));
if (plugins != null) {
// Initialize each plugins
for (String resourceName : plugins) {
@ -75,23 +82,21 @@ public synchronized void initialize(Context context)
String msg =
"Trying to initialize resource plugin with name=" + resourceName
+ ", it is not supported, list of supported plugins:"
+ StringUtils.join(",",
SUPPORTED_RESOURCE_PLUGINS);
+ StringUtils.join(",", SUPPORTED_RESOURCE_PLUGINS);
LOG.error(msg);
throw new YarnException(msg);
}
if (pluginMap.containsKey(resourceName)) {
// Duplicated items, ignore ...
LOG.warn("Ignoring duplicate Resource plugin definition: " +
resourceName);
continue;
}
ResourcePlugin plugin = null;
if (resourceName.equals(GPU_URI)) {
plugin = new GpuResourcePlugin();
}
if (resourceName.equals(FPGA_URI)) {
} else if (resourceName.equals(FPGA_URI)) {
plugin = new FpgaResourcePlugin();
}
@ -101,6 +106,7 @@ public synchronized void initialize(Context context)
+ " should be loaded and initialized");
}
plugin.initialize(context);
LOG.info("Initialized plugin {}", plugin);
pluginMap.put(resourceName, plugin);
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java
View File

@ -99,4 +99,9 @@ public NMResourceInfo getNMResourceInfo() throws YarnException {
public DeviceResourceHandlerImpl getDeviceResourceHandler() {
return deviceResourceHandler;
}
@Override
public String toString() {
return DevicePluginAdapter.class.getName();
}
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java
View File

@ -46,12 +46,12 @@ public class DeviceResourceHandlerImpl implements ResourceHandler {
static final Log LOG = LogFactory.getLog(DeviceResourceHandlerImpl.class);
private String resourceName;
private DevicePlugin devicePlugin;
private DeviceMappingManager deviceMappingManager;
private CGroupsHandler cGroupsHandler;
private PrivilegedOperationExecutor privilegedOperationExecutor;
private DevicePluginAdapter devicePluginAdapter;
private final String resourceName;
private final DevicePlugin devicePlugin;
private final DeviceMappingManager deviceMappingManager;
private final CGroupsHandler cGroupsHandler;
private final PrivilegedOperationExecutor privilegedOperationExecutor;
private final DevicePluginAdapter devicePluginAdapter;
public DeviceResourceHandlerImpl(String reseName,
DevicePlugin devPlugin,
@ -142,4 +142,13 @@ public List<PrivilegedOperation> teardown()
throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return DeviceResourceHandlerImpl.class.getName() + "{" +
"resourceName='" + resourceName + '\'' +
", devicePlugin=" + devicePlugin +
", devicePluginAdapter=" + devicePluginAdapter +
'}';
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaResourcePlugin.java
View File

@ -102,4 +102,9 @@ public DockerCommandPlugin getDockerCommandPluginInstance() {
public NMResourceInfo getNMResourceInfo() throws YarnException {
return null;
}
@Override
public String toString() {
return FpgaResourcePlugin.class.getName();
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
View File

@ -89,4 +89,9 @@ public NMResourceInfo getNMResourceInfo() throws YarnException {
return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus,
assignedGpuDevices);
}
@Override
public String toString() {
return GpuResourcePlugin.class.getName();
}
}