From 3541ed80685f25486f33ef0f553854ccbdeb51d4 Mon Sep 17 00:00:00 2001 From: Naganarasimha Date: Wed, 23 Nov 2016 14:12:23 +0530 Subject: [PATCH] YARN-4330. MiniYARNCluster is showing multiple Failed to instantiate default resource calculator warning messages. Contributed by Varun Saxena --- .../yarn/util/ResourceCalculatorPlugin.java | 3 + .../src/main/resources/yarn-default.xml | 6 +- .../nodemanager/NodeResourceMonitorImpl.java | 5 + .../monitor/ContainersMonitorImpl.java | 3 +- .../util/NodeManagerHardwareUtils.java | 160 ++++++++++-------- .../hadoop/yarn/server/MiniYARNCluster.java | 7 +- 6 files changed, 110 insertions(+), 74 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java index e7e4c8ae04..fd63d9852c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java @@ -190,6 +190,9 @@ public static ResourceCalculatorPlugin getResourceCalculatorPlugin( } try { return new ResourceCalculatorPlugin(); + } catch (UnsupportedOperationException ue) { + LOG.warn("Failed to instantiate default resource calculator. " + + ue.getMessage()); } catch (Throwable t) { LOG.warn(t + ": Failed to instantiate default resource calculator.", t); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c436289eaa..47d12d1626 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1344,7 +1344,8 @@ - How often to monitor the node and the containers. + How often to monitor the node and the containers. + If 0 or negative, monitoring is disabled. yarn.nodemanager.resource-monitor.interval-ms 3000 @@ -1362,7 +1363,8 @@ How often to monitor containers. If not set, the value for - yarn.nodemanager.resource-monitor.interval-ms will be used. + yarn.nodemanager.resource-monitor.interval-ms will be used. + If 0 or negative, container monitoring is disabled. yarn.nodemanager.container-monitor.interval-ms diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java index 8ee27ded24..e1116dadd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java @@ -78,6 +78,11 @@ protected void serviceInit(Configuration conf) throws Exception { * @return true if we can monitor the node resource utilization. */ private boolean isEnabled() { + if (this.monitoringInterval <= 0) { + LOG.info("Node Resource monitoring interval is <=0. " + + this.getClass().getName() + " is disabled."); + return false; + } if (resourceCalculatorPlugin == null) { LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this.getClass().getName() + " is disabled."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 1914b42ed2..dda55d24fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -172,7 +172,8 @@ protected void serviceInit(Configuration myConf) throws Exception { LOG.info("Physical memory check enabled: " + pmemCheckEnabled); LOG.info("Virtual memory check enabled: " + vmemCheckEnabled); - containersMonitorEnabled = isContainerMonitorEnabled(); + containersMonitorEnabled = + isContainerMonitorEnabled() && monitoringInterval > 0; LOG.info("ContainersMonitor enabled: " + containersMonitorEnabled); nodeCpuPercentageForYARN = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java index f3c95d3981..2726a41e5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java @@ -37,6 +37,12 @@ public class NodeManagerHardwareUtils { private static final Log LOG = LogFactory .getLog(NodeManagerHardwareUtils.class); + private static boolean isHardwareDetectionEnabled(Configuration conf) { + return conf.getBoolean( + YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, + YarnConfiguration.DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION); + } + /** * * Returns the number of CPUs on the node. This value depends on the @@ -138,6 +144,15 @@ public static int getNodeCpuPercentage(Configuration conf) { return nodeCpuPercentage; } + private static int getConfiguredVCores(Configuration conf) { + int cores = conf.getInt(YarnConfiguration.NM_VCORES, + YarnConfiguration.DEFAULT_NM_VCORES); + if (cores == -1) { + cores = YarnConfiguration.DEFAULT_NM_VCORES; + } + return cores; + } + /** * Function to return the number of vcores on the system that can be used for * YARN containers. If a number is specified in the configuration file, then @@ -154,11 +169,16 @@ public static int getNodeCpuPercentage(Configuration conf) { * */ public static int getVCores(Configuration conf) { + if (!isHardwareDetectionEnabled(conf)) { + return getConfiguredVCores(conf); + } // is this os for which we can determine cores? ResourceCalculatorPlugin plugin = ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf); - - return NodeManagerHardwareUtils.getVCores(plugin, conf); + if (plugin == null) { + return getConfiguredVCores(conf); + } + return getVCoresInternal(plugin, conf); } /** @@ -180,43 +200,35 @@ public static int getVCores(Configuration conf) { */ public static int getVCores(ResourceCalculatorPlugin plugin, Configuration conf) { + if (!isHardwareDetectionEnabled(conf) || plugin == null) { + return getConfiguredVCores(conf); + } + return getVCoresInternal(plugin, conf); + } - int cores; - boolean hardwareDetectionEnabled = - conf.getBoolean( - YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, - YarnConfiguration.DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION); - + private static int getVCoresInternal(ResourceCalculatorPlugin plugin, + Configuration conf) { String message; - if (!hardwareDetectionEnabled || plugin == null) { - cores = - conf.getInt(YarnConfiguration.NM_VCORES, - YarnConfiguration.DEFAULT_NM_VCORES); - if (cores == -1) { - cores = YarnConfiguration.DEFAULT_NM_VCORES; - } - } else { - cores = conf.getInt(YarnConfiguration.NM_VCORES, -1); - if (cores == -1) { - float physicalCores = - NodeManagerHardwareUtils.getContainersCPUs(plugin, conf); - float multiplier = - conf.getFloat(YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER, - YarnConfiguration.DEFAULT_NM_PCORES_VCORES_MULTIPLIER); - if (multiplier > 0) { - float tmp = physicalCores * multiplier; - if (tmp > 0 && tmp < 1) { - // on a single core machine - tmp can be between 0 and 1 - cores = 1; - } else { - cores = (int) tmp; - } + int cores = conf.getInt(YarnConfiguration.NM_VCORES, -1); + if (cores == -1) { + float physicalCores = + NodeManagerHardwareUtils.getContainersCPUs(plugin, conf); + float multiplier = + conf.getFloat(YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER, + YarnConfiguration.DEFAULT_NM_PCORES_VCORES_MULTIPLIER); + if (multiplier > 0) { + float tmp = physicalCores * multiplier; + if (tmp > 0 && tmp < 1) { + // on a single core machine - tmp can be between 0 and 1 + cores = 1; } else { - message = "Illegal value for " - + YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER - + ". Value must be greater than 0."; - throw new IllegalArgumentException(message); + cores = (int) tmp; } + } else { + message = "Illegal value for " + + YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER + + ". Value must be greater than 0."; + throw new IllegalArgumentException(message); } } if(cores <= 0) { @@ -228,6 +240,15 @@ public static int getVCores(ResourceCalculatorPlugin plugin, return cores; } + private static int getConfiguredMemoryMB(Configuration conf) { + int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, + YarnConfiguration.DEFAULT_NM_PMEM_MB); + if (memoryMb == -1) { + memoryMb = YarnConfiguration.DEFAULT_NM_PMEM_MB; + } + return memoryMb; + } + /** * Function to return how much memory we should set aside for YARN containers. * If a number is specified in the configuration file, then that number is @@ -244,8 +265,15 @@ public static int getVCores(ResourceCalculatorPlugin plugin, * @return the amount of memory that will be used for YARN containers in MB. */ public static int getContainerMemoryMB(Configuration conf) { - return NodeManagerHardwareUtils.getContainerMemoryMB( - ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf), conf); + if (!isHardwareDetectionEnabled(conf)) { + return getConfiguredMemoryMB(conf); + } + ResourceCalculatorPlugin plugin = + ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf); + if (plugin == null) { + return getConfiguredMemoryMB(conf); + } + return getContainerMemoryMBInternal(plugin, conf); } /** @@ -267,41 +295,35 @@ public static int getContainerMemoryMB(Configuration conf) { */ public static int getContainerMemoryMB(ResourceCalculatorPlugin plugin, Configuration conf) { + if (!isHardwareDetectionEnabled(conf) || plugin == null) { + return getConfiguredMemoryMB(conf); + } + return getContainerMemoryMBInternal(plugin, conf); + } - int memoryMb; - boolean hardwareDetectionEnabled = conf.getBoolean( - YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, - YarnConfiguration.DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION); - - if (!hardwareDetectionEnabled || plugin == null) { - memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, - YarnConfiguration.DEFAULT_NM_PMEM_MB); - if (memoryMb == -1) { - memoryMb = YarnConfiguration.DEFAULT_NM_PMEM_MB; + private static int getContainerMemoryMBInternal(ResourceCalculatorPlugin plugin, + Configuration conf) { + int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, -1); + if (memoryMb == -1) { + int physicalMemoryMB = + (int) (plugin.getPhysicalMemorySize() / (1024 * 1024)); + int hadoopHeapSizeMB = + (int) (Runtime.getRuntime().maxMemory() / (1024 * 1024)); + int containerPhysicalMemoryMB = + (int) (0.8f * (physicalMemoryMB - (2 * hadoopHeapSizeMB))); + int reservedMemoryMB = + conf.getInt(YarnConfiguration.NM_SYSTEM_RESERVED_PMEM_MB, -1); + if (reservedMemoryMB != -1) { + containerPhysicalMemoryMB = physicalMemoryMB - reservedMemoryMB; } - } else { - memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, -1); - if (memoryMb == -1) { - int physicalMemoryMB = - (int) (plugin.getPhysicalMemorySize() / (1024 * 1024)); - int hadoopHeapSizeMB = - (int) (Runtime.getRuntime().maxMemory() / (1024 * 1024)); - int containerPhysicalMemoryMB = - (int) (0.8f * (physicalMemoryMB - (2 * hadoopHeapSizeMB))); - int reservedMemoryMB = - conf.getInt(YarnConfiguration.NM_SYSTEM_RESERVED_PMEM_MB, -1); - if (reservedMemoryMB != -1) { - containerPhysicalMemoryMB = physicalMemoryMB - reservedMemoryMB; - } - if(containerPhysicalMemoryMB <= 0) { - LOG.error("Calculated memory for YARN containers is too low." - + " Node memory is " + physicalMemoryMB - + " MB, system reserved memory is " - + reservedMemoryMB + " MB."); - } - containerPhysicalMemoryMB = Math.max(containerPhysicalMemoryMB, 0); - memoryMb = containerPhysicalMemoryMB; + if(containerPhysicalMemoryMB <= 0) { + LOG.error("Calculated memory for YARN containers is too low." + + " Node memory is " + physicalMemoryMB + + " MB, system reserved memory is " + + reservedMemoryMB + " MB."); } + containerPhysicalMemoryMB = Math.max(containerPhysicalMemoryMB, 0); + memoryMb = containerPhysicalMemoryMB; } if(memoryMb <= 0) { String message = "Illegal value for " + YarnConfiguration.NM_PMEM_MB diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 89ebeb11a4..4f484ad329 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -552,13 +552,16 @@ protected synchronized void serviceInit(Configuration conf) .setNMWebAppHostNameAndPort(config, MiniYARNCluster.getHostname(), 0); + config.setBoolean( + YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, false); // Disable resource checks by default if (!config.getBoolean( YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, YarnConfiguration. DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) { - config.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false); - config.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false); + config.setBoolean( + YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED, false); + config.setLong(YarnConfiguration.NM_RESOURCE_MON_INTERVAL_MS, 0); } LOG.info("Starting NM: " + index);