From a6989af1d91d8b6795862c39d154823f8488c8de Mon Sep 17 00:00:00 2001 From: Sunil G Date: Thu, 17 Aug 2017 21:14:51 +0530 Subject: [PATCH] YARN-7030. [YARN-3926] Performance optimizations in Resource and ResourceUtils class. Contributed by Wangda Tan. --- .../hadoop/yarn/api/records/Resource.java | 6 ++--- .../yarn/api/records/impl/BaseResource.java | 10 +++++--- .../yarn/util/resource/ResourceUtils.java | 25 ++++++++++++++++--- .../hadoop/yarn/util/resource/Resources.java | 16 ++++++------ 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 1e9f213331..04579c53e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -65,7 +65,7 @@ public abstract class Resource implements Comparable { @Public @Stable public static Resource newInstance(int memory, int vCores) { - if (ResourceUtils.getResourceTypesArray().length > 2) { + if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { Resource ret = Records.newRecord(Resource.class); ret.setMemorySize(memory); ret.setVirtualCores(vCores); @@ -77,7 +77,7 @@ public static Resource newInstance(int memory, int vCores) { @Public @Stable public static Resource newInstance(long memory, int vCores) { - if (ResourceUtils.getResourceTypesArray().length > 2) { + if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { Resource ret = Records.newRecord(Resource.class); ret.setMemorySize(memory); ret.setVirtualCores(vCores); @@ -91,7 +91,7 @@ public static Resource newInstance(long memory, int vCores) { public static Resource newInstance(Resource resource) { Resource ret = Resource.newInstance(resource.getMemorySize(), resource.getVirtualCores()); - if (ResourceUtils.getResourceTypesArray().length > 2) { + if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { Resource.copy(resource, ret); } return ret; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java index 83db542a8a..b5cc4d63e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java @@ -63,6 +63,11 @@ public class BaseResource extends Resource { protected ResourceInformation[] resources = null; protected ResourceInformation[] readOnlyResources = null; + // Number of mandatory resources, this is added to avoid invoke + // MandatoryResources.values().length, since values() internally will + // copy array, etc. + private static final int NUM_MANDATORY_RESOURCES = 2; + protected enum MandatoryResources { MEMORY(0), VCORES(1); @@ -86,9 +91,8 @@ public BaseResource(long memory, long vcores) { ResourceInformation.MEMORY_MB.getUnits(), memory); this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores); - resources = new ResourceInformation[MandatoryResources.values().length]; - readOnlyResources = new ResourceInformation[MandatoryResources - .values().length]; + resources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; + readOnlyResources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; resources[MandatoryResources.MEMORY.id] = memoryResInfo; resources[MandatoryResources.VCORES.id] = vcoresResInfo; readOnlyResources = Arrays.copyOf(resources, resources.length); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 49feafa39d..06a541477a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -74,6 +74,7 @@ public class ResourceUtils { private static volatile ResourceInformation[] resourceTypesArray; private static volatile boolean initializedNodeResources = false; private static volatile Map readOnlyNodeResources; + private static volatile int numKnownResourceTypes = -1; static final Log LOG = LogFactory.getLog(ResourceUtils.class); @@ -308,23 +309,33 @@ public static Map getResourceTypes() { * @return resourceNamesArray */ public static String[] getResourceNamesArray() { - getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + initializeResourceTypesIfNeeded(null, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); return resourceNamesArray; } public static ResourceInformation[] getResourceTypesArray() { - getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + initializeResourceTypesIfNeeded(null, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); return resourceTypesArray; } + public static int getNumberOfKnownResourceTypes() { + if (numKnownResourceTypes < 0) { + initializeResourceTypesIfNeeded(null, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + } + return numKnownResourceTypes; + } + private static Map getResourceTypes( Configuration conf) { return getResourceTypes(conf, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); } - private static Map getResourceTypes( - Configuration conf, String resourceFile) { + private static void initializeResourceTypesIfNeeded(Configuration conf, + String resourceFile) { if (!initializedResources) { synchronized (ResourceUtils.class) { if (!initializedResources) { @@ -346,6 +357,12 @@ private static Map getResourceTypes( } } } + numKnownResourceTypes = resourceTypes.size(); + } + + private static Map getResourceTypes( + Configuration conf, String resourceFile) { + initializeResourceTypesIfNeeded(conf, resourceFile); return resourceTypes; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 702c1c8200..54ef0364a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -173,7 +173,7 @@ public static Resource clone(Resource res) { } public static Resource addTo(Resource lhs, Resource rhs) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -197,7 +197,7 @@ public static Resource add(Resource lhs, Resource rhs) { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -243,7 +243,7 @@ public static Resource negate(Resource resource) { } public static Resource multiplyTo(Resource lhs, double by) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); @@ -266,7 +266,7 @@ public static Resource multiply(Resource lhs, double by) { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -299,7 +299,7 @@ public static Resource multiplyAndNormalizeDown( public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); @@ -408,7 +408,7 @@ public static Resource max( } public static boolean fitsIn(Resource smaller, Resource bigger) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = bigger.getResourceInformation(i); @@ -436,7 +436,7 @@ public static boolean fitsIn(ResourceCalculator rc, Resource cluster, public static Resource componentwiseMin(Resource lhs, Resource rhs) { Resource ret = createResource(0); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -460,7 +460,7 @@ public static Resource componentwiseMin(Resource lhs, Resource rhs) { public static Resource componentwiseMax(Resource lhs, Resource rhs) { Resource ret = createResource(0); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i);