big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4330. MiniYARNCluster is showing multiple Failed to instantiate default resource calculator warning messages. Contributed by Varun Saxena

Browse Source
This commit is contained in:
Naganarasimha 2016-11-23 14:12:23 +05:30
parent 4667564162
commit 3541ed8068
6 changed files with 110 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
View File

@ -190,6 +190,9 @@ public static ResourceCalculatorPlugin getResourceCalculatorPlugin(
}
try {
return new ResourceCalculatorPlugin();
} catch (UnsupportedOperationException ue) {
LOG.warn("Failed to instantiate default resource calculator. "
+ ue.getMessage());
} catch (Throwable t) {
LOG.warn(t + ": Failed to instantiate default resource calculator.", t);
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1344,7 +1344,8 @@
</property>
<property>
<description>How often to monitor the node and the containers.</description>
<description>How often to monitor the node and the containers.
If 0 or negative, monitoring is disabled.</description>
<name>yarn.nodemanager.resource-monitor.interval-ms</name>
<value>3000</value>
</property>
@ -1362,7 +1363,8 @@
<property>
<description>How often to monitor containers. If not set, the value for
yarn.nodemanager.resource-monitor.interval-ms will be used.</description>
yarn.nodemanager.resource-monitor.interval-ms will be used.
If 0 or negative, container monitoring is disabled.</description>
<name>yarn.nodemanager.container-monitor.interval-ms</name>
</property>

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
View File

@ -78,6 +78,11 @@ protected void serviceInit(Configuration conf) throws Exception {
* @return <em>true</em> if we can monitor the node resource utilization.
*/
private boolean isEnabled() {
if (this.monitoringInterval <= 0) {
LOG.info("Node Resource monitoring interval is <=0. "
+ this.getClass().getName() + " is disabled.");
return false;
}
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
+ this.getClass().getName() + " is disabled.");

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
View File

@ -172,7 +172,8 @@ protected void serviceInit(Configuration myConf) throws Exception {
LOG.info("Physical memory check enabled: " + pmemCheckEnabled);
LOG.info("Virtual memory check enabled: " + vmemCheckEnabled);
containersMonitorEnabled = isContainerMonitorEnabled();
containersMonitorEnabled =
isContainerMonitorEnabled() && monitoringInterval > 0;
LOG.info("ContainersMonitor enabled: " + containersMonitorEnabled);
nodeCpuPercentageForYARN =

160
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
View File

@ -37,6 +37,12 @@ public class NodeManagerHardwareUtils {
private static final Log LOG = LogFactory
.getLog(NodeManagerHardwareUtils.class);
private static boolean isHardwareDetectionEnabled(Configuration conf) {
return conf.getBoolean(
YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION,
YarnConfiguration.DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION);
}
/**
*
* Returns the number of CPUs on the node. This value depends on the
@ -138,6 +144,15 @@ public static int getNodeCpuPercentage(Configuration conf) {
return nodeCpuPercentage;
}
private static int getConfiguredVCores(Configuration conf) {
int cores = conf.getInt(YarnConfiguration.NM_VCORES,
YarnConfiguration.DEFAULT_NM_VCORES);
if (cores == -1) {
cores = YarnConfiguration.DEFAULT_NM_VCORES;
}
return cores;
}
/**
* Function to return the number of vcores on the system that can be used for
* YARN containers. If a number is specified in the configuration file, then
@ -154,11 +169,16 @@ public static int getNodeCpuPercentage(Configuration conf) {
*
*/
public static int getVCores(Configuration conf) {
if (!isHardwareDetectionEnabled(conf)) {
return getConfiguredVCores(conf);
}
// is this os for which we can determine cores?
ResourceCalculatorPlugin plugin =
ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf);
return NodeManagerHardwareUtils.getVCores(plugin, conf);
if (plugin == null) {
return getConfiguredVCores(conf);
}
return getVCoresInternal(plugin, conf);
}
/**
@ -180,43 +200,35 @@ public static int getVCores(Configuration conf) {
*/
public static int getVCores(ResourceCalculatorPlugin plugin,
Configuration conf) {
if (!isHardwareDetectionEnabled(conf) || plugin == null) {
return getConfiguredVCores(conf);
}
return getVCoresInternal(plugin, conf);
}
int cores;
boolean hardwareDetectionEnabled =
conf.getBoolean(
YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION,
YarnConfiguration.DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION);
private static int getVCoresInternal(ResourceCalculatorPlugin plugin,
Configuration conf) {
String message;
if (!hardwareDetectionEnabled || plugin == null) {
cores =
conf.getInt(YarnConfiguration.NM_VCORES,
YarnConfiguration.DEFAULT_NM_VCORES);
if (cores == -1) {
cores = YarnConfiguration.DEFAULT_NM_VCORES;
}
} else {
cores = conf.getInt(YarnConfiguration.NM_VCORES, -1);
if (cores == -1) {
float physicalCores =
NodeManagerHardwareUtils.getContainersCPUs(plugin, conf);
float multiplier =
conf.getFloat(YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER,
YarnConfiguration.DEFAULT_NM_PCORES_VCORES_MULTIPLIER);
if (multiplier > 0) {
float tmp = physicalCores * multiplier;
if (tmp > 0 && tmp < 1) {
// on a single core machine - tmp can be between 0 and 1
cores = 1;
} else {
cores = (int) tmp;
}
int cores = conf.getInt(YarnConfiguration.NM_VCORES, -1);
if (cores == -1) {
float physicalCores =
NodeManagerHardwareUtils.getContainersCPUs(plugin, conf);
float multiplier =
conf.getFloat(YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER,
YarnConfiguration.DEFAULT_NM_PCORES_VCORES_MULTIPLIER);
if (multiplier > 0) {
float tmp = physicalCores * multiplier;
if (tmp > 0 && tmp < 1) {
// on a single core machine - tmp can be between 0 and 1
cores = 1;
} else {
message = "Illegal value for "
+ YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER
+ ". Value must be greater than 0.";
throw new IllegalArgumentException(message);
cores = (int) tmp;
}
} else {
message = "Illegal value for "
+ YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER
+ ". Value must be greater than 0.";
throw new IllegalArgumentException(message);
}
}
if(cores <= 0) {
@ -228,6 +240,15 @@ public static int getVCores(ResourceCalculatorPlugin plugin,
return cores;
}
private static int getConfiguredMemoryMB(Configuration conf) {
int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB,
YarnConfiguration.DEFAULT_NM_PMEM_MB);
if (memoryMb == -1) {
memoryMb = YarnConfiguration.DEFAULT_NM_PMEM_MB;
}
return memoryMb;
}
/**
* Function to return how much memory we should set aside for YARN containers.
* If a number is specified in the configuration file, then that number is
@ -244,8 +265,15 @@ public static int getVCores(ResourceCalculatorPlugin plugin,
* @return the amount of memory that will be used for YARN containers in MB.
*/
public static int getContainerMemoryMB(Configuration conf) {
return NodeManagerHardwareUtils.getContainerMemoryMB(
ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf), conf);
if (!isHardwareDetectionEnabled(conf)) {
return getConfiguredMemoryMB(conf);
}
ResourceCalculatorPlugin plugin =
ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf);
if (plugin == null) {
return getConfiguredMemoryMB(conf);
}
return getContainerMemoryMBInternal(plugin, conf);
}
/**
@ -267,41 +295,35 @@ public static int getContainerMemoryMB(Configuration conf) {
*/
public static int getContainerMemoryMB(ResourceCalculatorPlugin plugin,
Configuration conf) {
if (!isHardwareDetectionEnabled(conf) || plugin == null) {
return getConfiguredMemoryMB(conf);
}
return getContainerMemoryMBInternal(plugin, conf);
}
int memoryMb;
boolean hardwareDetectionEnabled = conf.getBoolean(
YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION,
YarnConfiguration.DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION);
if (!hardwareDetectionEnabled || plugin == null) {
memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB,
YarnConfiguration.DEFAULT_NM_PMEM_MB);
if (memoryMb == -1) {
memoryMb = YarnConfiguration.DEFAULT_NM_PMEM_MB;
private static int getContainerMemoryMBInternal(ResourceCalculatorPlugin plugin,
Configuration conf) {
int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, -1);
if (memoryMb == -1) {
int physicalMemoryMB =
(int) (plugin.getPhysicalMemorySize() / (1024 * 1024));
int hadoopHeapSizeMB =
(int) (Runtime.getRuntime().maxMemory() / (1024 * 1024));
int containerPhysicalMemoryMB =
(int) (0.8f * (physicalMemoryMB - (2 * hadoopHeapSizeMB)));
int reservedMemoryMB =
conf.getInt(YarnConfiguration.NM_SYSTEM_RESERVED_PMEM_MB, -1);
if (reservedMemoryMB != -1) {
containerPhysicalMemoryMB = physicalMemoryMB - reservedMemoryMB;
}
} else {
memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, -1);
if (memoryMb == -1) {
int physicalMemoryMB =
(int) (plugin.getPhysicalMemorySize() / (1024 * 1024));
int hadoopHeapSizeMB =
(int) (Runtime.getRuntime().maxMemory() / (1024 * 1024));
int containerPhysicalMemoryMB =
(int) (0.8f * (physicalMemoryMB - (2 * hadoopHeapSizeMB)));
int reservedMemoryMB =
conf.getInt(YarnConfiguration.NM_SYSTEM_RESERVED_PMEM_MB, -1);
if (reservedMemoryMB != -1) {
containerPhysicalMemoryMB = physicalMemoryMB - reservedMemoryMB;
}
if(containerPhysicalMemoryMB <= 0) {
LOG.error("Calculated memory for YARN containers is too low."
+ " Node memory is " + physicalMemoryMB
+ " MB, system reserved memory is "
+ reservedMemoryMB + " MB.");
}
containerPhysicalMemoryMB = Math.max(containerPhysicalMemoryMB, 0);
memoryMb = containerPhysicalMemoryMB;
if(containerPhysicalMemoryMB <= 0) {
LOG.error("Calculated memory for YARN containers is too low."
+ " Node memory is " + physicalMemoryMB
+ " MB, system reserved memory is "
+ reservedMemoryMB + " MB.");
}
containerPhysicalMemoryMB = Math.max(containerPhysicalMemoryMB, 0);
memoryMb = containerPhysicalMemoryMB;
}
if(memoryMb <= 0) {
String message = "Illegal value for " + YarnConfiguration.NM_PMEM_MB

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
View File

@ -552,13 +552,16 @@ protected synchronized void serviceInit(Configuration conf)
.setNMWebAppHostNameAndPort(config,
MiniYARNCluster.getHostname(), 0);
config.setBoolean(
YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, false);
// Disable resource checks by default
if (!config.getBoolean(
YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
YarnConfiguration.
DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
config.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
config.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
config.setBoolean(
YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED, false);
config.setLong(YarnConfiguration.NM_RESOURCE_MON_INTERVAL_MS, 0);
}
LOG.info("Starting NM: " + index);