From 759114b0063907d4c07ea6ee261e861bf5cc3a9a Mon Sep 17 00:00:00 2001 From: Varun Vasudev Date: Sat, 11 Jun 2016 14:33:46 +0530 Subject: [PATCH] YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev. --- .../hadoop/yarn/api/records/Resource.java | 3 +- .../hadoop/yarn/conf/YarnConfiguration.java | 19 +- .../FileSystemBasedConfigurationProvider.java | 3 +- .../yarn/LocalConfigurationProvider.java | 3 +- .../api/records/impl/pb/ResourcePBImpl.java | 53 +++--- .../yarn/util/resource/ResourceUtils.java | 168 ++++++++++++++---- .../yarn/util/resource/TestResourceUtils.java | 29 ++- .../resource-types/node-resources-1.xml | 29 +++ .../resource-types/node-resources-2.xml | 39 ++++ .../nodemanager/NodeStatusUpdaterImpl.java | 7 +- .../util/NodeManagerHardwareUtils.java | 52 ++++++ .../ResourceTrackerService.java | 9 +- 12 files changed, 342 insertions(+), 72 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-1.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index ee8ef0324f..c9c6a7ae55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -308,7 +308,8 @@ public String toString() { continue; } if (entry.getKey().equals(ResourceInformation.VCORES.getName()) - && entry.getValue().getUnits().equals("")) { + && entry.getValue().getUnits() + .equals(ResourceInformation.VCORES.getUnits())) { continue; } sb.append(", ").append(entry.getKey()).append(": ") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 318658f8c3..b3095fc0fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -64,6 +64,10 @@ public class YarnConfiguration extends Configuration { public static final String RESOURCE_TYPES_CONFIGURATION_FILE = "resource-types.xml"; + @Private + public static final String NODE_RESOURCES_CONFIGURATION_FILE = + "node-resources.xml"; + @Private public static final List RM_CONFIGURATION_FILES = Collections.unmodifiableList(Arrays.asList( @@ -74,6 +78,16 @@ public class YarnConfiguration extends Configuration { YARN_SITE_CONFIGURATION_FILE, CORE_SITE_CONFIGURATION_FILE)); + @Private + public static final List NM_CONFIGURATION_FILES = + Collections.unmodifiableList(Arrays.asList( + NODE_RESOURCES_CONFIGURATION_FILE, + DR_CONFIGURATION_FILE, + CS_CONFIGURATION_FILE, + HADOOP_POLICY_CONFIGURATION_FILE, + YARN_SITE_CONFIGURATION_FILE, + CORE_SITE_CONFIGURATION_FILE)); + @Evolving public static final int APPLICATION_MAX_TAGS = 10; @@ -112,12 +126,15 @@ private static void addDeprecatedKeys() { public static final String YARN_PREFIX = "yarn."; ///////////////////////////// - // Scheduler resource types configs + // Resource types configs //////////////////////////// public static final String RESOURCE_TYPES = YarnConfiguration.YARN_PREFIX + "resource-types"; + public static final String NM_RESOURCES_PREFIX = + YarnConfiguration.NM_PREFIX + "resource-type."; + /** Delay before deleting resource to ease debugging of NM issues */ public static final String DEBUG_NM_DELETE_DELAY_SEC = YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java index cef03b9b05..73c49906c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -51,7 +51,8 @@ public synchronized InputStream getConfigurationInputStream( "Illegal argument! The parameter should not be null or empty"); } Path filePath; - if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) { + if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name) || + YarnConfiguration.NM_CONFIGURATION_FILES.contains(name)) { filePath = new Path(this.configDir, name); if (!fs.exists(filePath)) { LOG.info(filePath + " not found"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java index cfa194fb5b..0cdbd1516d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java @@ -39,7 +39,8 @@ public InputStream getConfigurationInputStream(Configuration bootstrapConf, if (name == null || name.isEmpty()) { throw new YarnException( "Illegal argument! The parameter should not be null or empty"); - } else if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) { + } else if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name) || + YarnConfiguration.NM_CONFIGURATION_FILES.contains(name)) { return bootstrapConf.getConfResourceAsInputStream(name); } return new FileInputStream(name); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 174f3cc5b8..86ae41f379 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -18,7 +18,8 @@ package org.apache.hadoop.yarn.api.records.impl.pb; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; @@ -38,6 +39,8 @@ @Unstable public class ResourcePBImpl extends Resource { + private static final Log LOG = LogFactory.getLog(ResourcePBImpl.class); + ResourceProto proto = ResourceProto.getDefaultInstance(); ResourceProto.Builder builder = null; boolean viaProto = false; @@ -92,10 +95,12 @@ public int getMemory() { @Override public long getMemorySize() { // memory should always be present - initResourcesMap(); + initResources(); ResourceInformation ri = this.getResourceInformation(ResourceInformation.MEMORY_MB.getName()); - return UnitsConversionUtil.convert(ri.getUnits(), "Mi", ri.getValue()); + return UnitsConversionUtil + .convert(ri.getUnits(), ResourceInformation.MEMORY_MB.getUnits(), + ri.getValue()); } @Override @@ -115,21 +120,16 @@ public void setMemorySize(long memory) { @Override public int getVirtualCores() { // vcores should always be present - initResourcesMap(); + initResources(); return this.getResourceValue(ResourceInformation.VCORES.getName()) .intValue(); } @Override public void setVirtualCores(int vCores) { - try { - setResourceValue(ResourceInformation.VCORES.getName(), - Long.valueOf(vCores)); - } catch (ResourceNotFoundException re) { - this.setResourceInformation(ResourceInformation.VCORES.getName(), - ResourceInformation.newInstance(ResourceInformation.VCORES.getName(), - (long) vCores)); - } + setResourceInformation(ResourceInformation.VCORES.getName(), + ResourceInformation.newInstance(ResourceInformation.VCORES.getName(), + ResourceInformation.VCORES.getUnits(), (long) vCores)); } private void initResources() { @@ -146,14 +146,16 @@ private void initResources() { Long value = entry.hasValue() ? entry.getValue() : 0L; ResourceInformation ri = ResourceInformation.newInstance(entry.getKey(), units, value, type); - resources.put(ri.getName(), ri); - } - if(this.getMemory() != p.getMemory()) { - setMemorySize(p.getMemory()); - } - if(this.getVirtualCores() != p.getVirtualCores()) { - setVirtualCores(p.getVirtualCores()); + if (resources.containsKey(ri.getName())) { + resources.get(ri.getName()).setResourceType(ri.getResourceType()); + resources.get(ri.getName()).setUnits(ri.getUnits()); + resources.get(ri.getName()).setValue(value); + } else { + LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping"); + } } + this.setMemorySize(p.getMemory()); + this.setVirtualCores(p.getVirtualCores()); } @Override @@ -167,7 +169,7 @@ public void setResourceInformation(String resource, if (!resource.equals(resourceInformation.getName())) { resourceInformation.setName(resource); } - initResourcesMap(); + initResources(); resources.put(resource, resourceInformation); } @@ -175,6 +177,7 @@ public void setResourceInformation(String resource, public void setResourceValue(String resource, Long value) throws ResourceNotFoundException { maybeInitBuilder(); + initResources(); if (resource == null) { throw new IllegalArgumentException("resource type object cannot be null"); } @@ -182,9 +185,7 @@ public void setResourceValue(String resource, Long value) throw new ResourceNotFoundException( "Resource " + resource + " not found"); } - ResourceInformation ri = resources.get(resource); - ri.setValue(value); - resources.put(resource, ri); + resources.get(resource).setValue(value); } @Override @@ -229,8 +230,10 @@ private void initResourcesMap() { synchronized private void mergeLocalToBuilder() { builder.clearResourceValueMap(); if (resources != null && !resources.isEmpty()) { - for (Map.Entry entry : resources.entrySet()) { - ResourceInformationProto.Builder e = ResourceInformationProto.newBuilder(); + for (Map.Entry entry : + resources.entrySet()) { + ResourceInformationProto.Builder e = + ResourceInformationProto.newBuilder(); e.setKey(entry.getKey()); e.setUnits(entry.getValue().getUnits()); e.setType( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 0316f010e8..4f56fbb52b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.util.resource; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -51,15 +52,21 @@ public class ResourceUtils { public static final String UNITS = ".units"; public static final String TYPE = ".type"; + private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); + private static final String VCORES = ResourceInformation.VCORES.getName(); + private static final Set DISALLOWED_NAMES = new HashSet<>(); static { DISALLOWED_NAMES.add("memory"); - DISALLOWED_NAMES.add(ResourceInformation.MEMORY_MB.getName()); - DISALLOWED_NAMES.add(ResourceInformation.VCORES.getName()); + DISALLOWED_NAMES.add(MEMORY); + DISALLOWED_NAMES.add(VCORES); } private static volatile Object lock; private static Map readOnlyResources; + private static volatile Object nodeLock; + private static Map readOnlyNodeResources; + static final Log LOG = LogFactory.getLog(ResourceUtils.class); @@ -69,22 +76,20 @@ private ResourceUtils() { private static void checkMandatatoryResources( Map resourceInformationMap) throws YarnRuntimeException { - String memory = ResourceInformation.MEMORY_MB.getName(); - String vcores = ResourceInformation.VCORES.getName(); - if (resourceInformationMap.containsKey(memory)) { - ResourceInformation memInfo = resourceInformationMap.get(memory); + if (resourceInformationMap.containsKey(MEMORY)) { + ResourceInformation memInfo = resourceInformationMap.get(MEMORY); String memUnits = ResourceInformation.MEMORY_MB.getUnits(); ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType(); if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType() .equals(memType)) { throw new YarnRuntimeException( "Attempt to re-define mandatory resource 'memory-mb'. It can only" - + " be of type 'COUNTABLE' and have units 'M'."); + + " be of type 'COUNTABLE' and have units 'Mi'."); } } - if (resourceInformationMap.containsKey(vcores)) { - ResourceInformation vcoreInfo = resourceInformationMap.get(vcores); + if (resourceInformationMap.containsKey(VCORES)) { + ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES); String vcoreUnits = ResourceInformation.VCORES.getUnits(); ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType(); if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo @@ -99,21 +104,21 @@ private static void checkMandatatoryResources( private static void addManadtoryResources( Map res) { ResourceInformation ri; - if (!res.containsKey(ResourceInformation.MEMORY_MB.getName())) { - LOG.info("Adding resource type - name = " + ResourceInformation.MEMORY_MB - .getName() + ", units = " + ResourceInformation.MEMORY_MB.getUnits() - + ", type = " + ResourceTypes.COUNTABLE); + if (!res.containsKey(MEMORY)) { + LOG.info("Adding resource type - name = " + MEMORY + ", units = " + + ResourceInformation.MEMORY_MB.getUnits() + ", type = " + + ResourceTypes.COUNTABLE); ri = ResourceInformation - .newInstance(ResourceInformation.MEMORY_MB.getName(), + .newInstance(MEMORY, ResourceInformation.MEMORY_MB.getUnits()); - res.put(ResourceInformation.MEMORY_MB.getName(), ri); + res.put(MEMORY, ri); } - if (!res.containsKey(ResourceInformation.VCORES.getName())) { - LOG.info("Adding resource type - name = " + ResourceInformation.VCORES - .getName() + ", units = , type = " + ResourceTypes.COUNTABLE); + if (!res.containsKey(VCORES)) { + LOG.info("Adding resource type - name = " + VCORES + ", units = , type = " + + ResourceTypes.COUNTABLE); ri = - ResourceInformation.newInstance(ResourceInformation.VCORES.getName()); - res.put(ResourceInformation.VCORES.getName(), ri); + ResourceInformation.newInstance(VCORES); + res.put(VCORES, ri); } } @@ -122,6 +127,7 @@ static void initializeResourcesMap(Configuration conf, Map resourceInformationMap) { String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES); + if (resourceNames != null && resourceNames.length != 0) { for (String resourceName : resourceNames) { String resourceUnits = conf.get( @@ -178,25 +184,13 @@ private static Map getResourceTypes( conf = new YarnConfiguration(); } try { - InputStream ris = getConfInputStream(resourceFile, conf); + addResourcesFileToConf(resourceFile, conf); LOG.debug("Found " + resourceFile + ", adding to configuration"); - conf.addResource(ris); initializeResourcesMap(conf, resources); - return resources; } catch (FileNotFoundException fe) { LOG.info("Unable to find '" + resourceFile + "'. Falling back to memory and vcores as resources", fe); initializeResourcesMap(conf, resources); - } catch (IOException ie) { - LOG.fatal( - "Exception trying to read resource types configuration '" - + resourceFile + "'.", ie); - throw new YarnRuntimeException(ie); - } catch (YarnException ye) { - LOG.fatal( - "YARN Exception trying to read resource types configuration '" - + resourceFile + "'.", ye); - throw new YarnRuntimeException(ye); } } } @@ -205,8 +199,8 @@ private static Map getResourceTypes( return readOnlyResources; } - static InputStream getConfInputStream(String resourceFile, Configuration conf) - throws IOException, YarnException { + private static InputStream getConfInputStream(String resourceFile, + Configuration conf) throws IOException, YarnException { ConfigurationProvider provider = ConfigurationProviderFactory.getConfigurationProvider(conf); @@ -222,8 +216,112 @@ static InputStream getConfInputStream(String resourceFile, Configuration conf) return ris; } + private static void addResourcesFileToConf(String resourceFile, + Configuration conf) throws FileNotFoundException { + try { + InputStream ris = getConfInputStream(resourceFile, conf); + LOG.debug("Found " + resourceFile + ", adding to configuration"); + conf.addResource(ris); + } catch (FileNotFoundException fe) { + throw fe; + } catch (IOException ie) { + LOG.fatal("Exception trying to read resource types configuration '" + + resourceFile + "'.", ie); + throw new YarnRuntimeException(ie); + } catch (YarnException ye) { + LOG.fatal("YARN Exception trying to read resource types configuration '" + + resourceFile + "'.", ye); + throw new YarnRuntimeException(ye); + } + } + @VisibleForTesting static void resetResourceTypes() { lock = null; } + + private static String getUnits(String resourceValue) { + String units; + for (int i = 0; i < resourceValue.length(); i++) { + if (Character.isAlphabetic(resourceValue.charAt(i))) { + units = resourceValue.substring(i); + if (StringUtils.isAlpha(units)) { + return units; + } + } + } + return ""; + } + + /** + * Function to get the resources for a node. This function will look at the + * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to + * determine the node resources. + * + * @param conf configuration file + * @return a map to resource name to the ResourceInformation object. The map + * is guaranteed to have entries for memory and vcores + */ + public static Map getNodeResourceInformation( + Configuration conf) { + if (nodeLock == null) { + synchronized (ResourceUtils.class) { + if (nodeLock == null) { + synchronized (ResourceUtils.class) { + nodeLock = new Object(); + Map nodeResources = + initializeNodeResourceInformation(conf); + addManadtoryResources(nodeResources); + checkMandatatoryResources(nodeResources); + readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); + } + } + } + } + return readOnlyNodeResources; + } + + private static Map + initializeNodeResourceInformation(Configuration conf) { + Map nodeResources = new HashMap<>(); + try { + addResourcesFileToConf( + YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf); + for (Map.Entry entry : conf) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) { + addResourceInformation(key, value, nodeResources); + } + } + } catch (FileNotFoundException fe) { + LOG.info("Couldn't find node resources file"); + } + return nodeResources; + } + + private static void addResourceInformation(String prop, String value, + Map nodeResources) { + String[] parts = prop.split("\\."); + LOG.info("Found resource entry " + prop); + if (parts.length == 4) { + String resourceType = parts[3]; + if (!nodeResources.containsKey(resourceType)) { + nodeResources + .put(resourceType, ResourceInformation.newInstance(resourceType)); + } + String units = getUnits(value); + Long resourceValue = + Long.valueOf(value.substring(0, value.length() - units.length())); + nodeResources.get(resourceType).setValue(resourceValue); + nodeResources.get(resourceType).setUnits(units); + LOG.debug("Setting value for resource type " + resourceType + " to " + + resourceValue + " with units " + units); + } + } + + @VisibleForTesting + synchronized public static void resetNodeResources() { + nodeLock = null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 4bb68c002a..ab014dd1c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -21,9 +21,9 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.util.SystemClock; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -245,4 +245,31 @@ public void testInitializeResourcesMapErrors() throws Exception { } } } + + @Test + public void testGetResourceInformation() throws Exception { + + Configuration conf = new YarnConfiguration(); + Map testRun = new HashMap<>(); + // testRun.put("node-resources-1.xml", Resource.newInstance(1024, 1)); + Resource test3Resources = Resource.newInstance(1024, 1); + test3Resources.setResourceInformation("resource1", + ResourceInformation.newInstance("resource1", "Gi", 5L)); + test3Resources.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 2L)); + testRun.put("node-resources-2.xml", test3Resources); + + for (Map.Entry entry : testRun.entrySet()) { + String resourceFile = entry.getKey(); + ResourceUtils.resetNodeResources(); + File dest; + File source = + new File(conf.getClassLoader().getResource(resourceFile).getFile()); + dest = new File(source.getParent(), "node-resources.xml"); + FileUtils.copyFile(source, dest); + Map actual = + ResourceUtils.getNodeResourceInformation(conf); + Assert.assertEquals(entry.getValue().getResources(), actual); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-1.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-1.xml new file mode 100644 index 0000000000..f00573e307 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-1.xml @@ -0,0 +1,29 @@ + + + + + + + + yarn.nodemanager.resource.memory-mb + 1024 + + + + yarn.nodemanager.resource.vcores + 1 + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml new file mode 100644 index 0000000000..9d9b3dc65c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml @@ -0,0 +1,39 @@ + + + + + + + + yarn.nodemanager.resource-type.memory-mb + 1024Mi + + + + yarn.nodemanager.resource-type.vcores + 1 + + + + yarn.nodemanager.resource-type.resource1 + 5Gi + + + + yarn.nodemanager.resource-type.resource2 + 2m + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 35b7cb0e5f..df56f38631 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -174,18 +174,19 @@ public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher, @Override protected void serviceInit(Configuration conf) throws Exception { - int memoryMb = NodeManagerHardwareUtils.getContainerMemoryMB(conf); + this.totalResource = NodeManagerHardwareUtils.getNodeResources(conf); + int memoryMb = totalResource.getMemory(); float vMemToPMem = conf.getFloat( YarnConfiguration.NM_VMEM_PMEM_RATIO, YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); int virtualMemoryMb = (int)Math.ceil(memoryMb * vMemToPMem); - int virtualCores = NodeManagerHardwareUtils.getVCores(conf); + int virtualCores = totalResource.getVirtualCores(); LOG.info("Nodemanager resources: memory set to " + memoryMb + "MB."); LOG.info("Nodemanager resources: vcores set to " + virtualCores + "."); + LOG.info("Nodemanager resources: " + totalResource); - this.totalResource = Resource.newInstance(memoryMb, virtualCores); metrics.addResource(totalResource); // Get actual node physical resources diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java index 32f73c85a0..40f6fade73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java @@ -21,10 +21,16 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; + +import java.util.Map; /** * Helper class to determine hardware related characteristics such as the @@ -332,4 +338,50 @@ private static int getContainerMemoryMBInternal(ResourceCalculatorPlugin plugin, } return memoryMb; } + + /** + * Get the resources for the node. + * @param configuration configuration file + * @return the resources for the node + */ + public static Resource getNodeResources(Configuration configuration) { + Configuration conf = new Configuration(configuration); + String memory = ResourceInformation.MEMORY_MB.getName(); + String vcores = ResourceInformation.VCORES.getName(); + + Resource ret = Resource.newInstance(0, 0); + Map resourceInformation = + ResourceUtils.getNodeResourceInformation(conf); + for (Map.Entry entry : resourceInformation + .entrySet()) { + ret.setResourceInformation(entry.getKey(), entry.getValue()); + LOG.debug("Setting key " + entry.getKey() + " to " + entry.getValue()); + } + if (resourceInformation.containsKey(memory)) { + Long value = resourceInformation.get(memory).getValue(); + if (value > Integer.MAX_VALUE) { + throw new YarnRuntimeException("Value '" + value + + "' for resource memory is more than the maximum for an integer."); + } + ResourceInformation memResInfo = resourceInformation.get(memory); + if(memResInfo.getValue() == 0) { + ret.setMemory(getContainerMemoryMB(conf)); + LOG.debug("Set memory to " + ret.getMemory()); + } + } + if (resourceInformation.containsKey(vcores)) { + Long value = resourceInformation.get(vcores).getValue(); + if (value > Integer.MAX_VALUE) { + throw new YarnRuntimeException("Value '" + value + + "' for resource vcores is more than the maximum for an integer."); + } + ResourceInformation vcoresResInfo = resourceInformation.get(vcores); + if(vcoresResInfo.getValue() == 0) { + ret.setVirtualCores(getVCores(conf)); + LOG.debug("Set vcores to " + ret.getVirtualCores()); + } + } + LOG.debug("Node resource information map is " + ret); + return ret; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index cc47e02cb1..a42d0533c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -376,10 +376,11 @@ public RegisterNodeManagerResponse registerNodeManager( // Check if this node has minimum allocations if (capability.getMemorySize() < minAllocMb || capability.getVirtualCores() < minAllocVcores) { - String message = - "NodeManager from " + host - + " doesn't satisfy minimum allocations, Sending SHUTDOWN" - + " signal to the NodeManager."; + String message = "NodeManager from " + host + + " doesn't satisfy minimum allocations, Sending SHUTDOWN" + + " signal to the NodeManager. Node capabilities are " + capability + + "; minimums are " + minAllocMb + "mb and " + minAllocVcores + + " vcores"; LOG.info(message); response.setDiagnosticsMessage(message); response.setNodeAction(NodeAction.SHUTDOWN);