big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7030. [YARN-3926] Performance optimizations in Resource and ResourceUtils class. Contributed by Wangda Tan.

Browse Source
This commit is contained in:
Sunil G 2017-08-17 21:14:51 +05:30 committed by Wangda Tan
parent a0030c8c6f
commit a6989af1d9
4 changed files with 39 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
View File

@ -65,7 +65,7 @@ public abstract class Resource implements Comparable<Resource> {
@Public @Public
@Stable @Stable
public static Resource newInstance(int memory, int vCores) { public static Resource newInstance(int memory, int vCores) {
if (ResourceUtils.getResourceTypesArray().length > 2) { if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
Resource ret = Records.newRecord(Resource.class); Resource ret = Records.newRecord(Resource.class);
ret.setMemorySize(memory); ret.setMemorySize(memory);
ret.setVirtualCores(vCores); ret.setVirtualCores(vCores);
@ -77,7 +77,7 @@ public static Resource newInstance(int memory, int vCores) {
@Public @Public
@Stable @Stable
public static Resource newInstance(long memory, int vCores) { public static Resource newInstance(long memory, int vCores) {
if (ResourceUtils.getResourceTypesArray().length > 2) { if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
Resource ret = Records.newRecord(Resource.class); Resource ret = Records.newRecord(Resource.class);
ret.setMemorySize(memory); ret.setMemorySize(memory);
ret.setVirtualCores(vCores); ret.setVirtualCores(vCores);
@ -91,7 +91,7 @@ public static Resource newInstance(long memory, int vCores) {
public static Resource newInstance(Resource resource) { public static Resource newInstance(Resource resource) {
Resource ret = Resource.newInstance(resource.getMemorySize(), Resource ret = Resource.newInstance(resource.getMemorySize(),
resource.getVirtualCores()); resource.getVirtualCores());
if (ResourceUtils.getResourceTypesArray().length > 2) { if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
Resource.copy(resource, ret); Resource.copy(resource, ret);
} }
return ret; return ret;

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
View File

@ -63,6 +63,11 @@ public class BaseResource extends Resource {
protected ResourceInformation[] resources = null; protected ResourceInformation[] resources = null;
protected ResourceInformation[] readOnlyResources = null; protected ResourceInformation[] readOnlyResources = null;
// Number of mandatory resources, this is added to avoid invoke
// MandatoryResources.values().length, since values() internally will
// copy array, etc.
private static final int NUM_MANDATORY_RESOURCES = 2;
protected enum MandatoryResources { protected enum MandatoryResources {
MEMORY(0), VCORES(1); MEMORY(0), VCORES(1);
@ -86,9 +91,8 @@ public BaseResource(long memory, long vcores) {
ResourceInformation.MEMORY_MB.getUnits(), memory); ResourceInformation.MEMORY_MB.getUnits(), memory);
this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores); this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores);
resources = new ResourceInformation[MandatoryResources.values().length]; resources = new ResourceInformation[NUM_MANDATORY_RESOURCES];
readOnlyResources = new ResourceInformation[MandatoryResources readOnlyResources = new ResourceInformation[NUM_MANDATORY_RESOURCES];
.values().length];
resources[MandatoryResources.MEMORY.id] = memoryResInfo; resources[MandatoryResources.MEMORY.id] = memoryResInfo;
resources[MandatoryResources.VCORES.id] = vcoresResInfo; resources[MandatoryResources.VCORES.id] = vcoresResInfo;
readOnlyResources = Arrays.copyOf(resources, resources.length); readOnlyResources = Arrays.copyOf(resources, resources.length);

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
View File

@ -74,6 +74,7 @@ public class ResourceUtils {
private static volatile ResourceInformation[] resourceTypesArray; private static volatile ResourceInformation[] resourceTypesArray;
private static volatile boolean initializedNodeResources = false; private static volatile boolean initializedNodeResources = false;
private static volatile Map<String, ResourceInformation> readOnlyNodeResources; private static volatile Map<String, ResourceInformation> readOnlyNodeResources;
private static volatile int numKnownResourceTypes = -1;
static final Log LOG = LogFactory.getLog(ResourceUtils.class); static final Log LOG = LogFactory.getLog(ResourceUtils.class);
@ -308,23 +309,33 @@ public static Map<String, ResourceInformation> getResourceTypes() {
* @return resourceNamesArray * @return resourceNamesArray
*/ */
public static String[] getResourceNamesArray() { public static String[] getResourceNamesArray() {
getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); initializeResourceTypesIfNeeded(null,
YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
return resourceNamesArray; return resourceNamesArray;
} }
public static ResourceInformation[] getResourceTypesArray() { public static ResourceInformation[] getResourceTypesArray() {
getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); initializeResourceTypesIfNeeded(null,
YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
return resourceTypesArray; return resourceTypesArray;
} }
public static int getNumberOfKnownResourceTypes() {
if (numKnownResourceTypes < 0) {
initializeResourceTypesIfNeeded(null,
YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
}
return numKnownResourceTypes;
}
private static Map<String, ResourceInformation> getResourceTypes( private static Map<String, ResourceInformation> getResourceTypes(
Configuration conf) { Configuration conf) {
return getResourceTypes(conf, return getResourceTypes(conf,
YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
} }
private static Map<String, ResourceInformation> getResourceTypes( private static void initializeResourceTypesIfNeeded(Configuration conf,
Configuration conf, String resourceFile) { String resourceFile) {
if (!initializedResources) { if (!initializedResources) {
synchronized (ResourceUtils.class) { synchronized (ResourceUtils.class) {
if (!initializedResources) { if (!initializedResources) {
@ -346,6 +357,12 @@ private static Map<String, ResourceInformation> getResourceTypes(
} }
} }
} }
numKnownResourceTypes = resourceTypes.size();
}
private static Map<String, ResourceInformation> getResourceTypes(
Configuration conf, String resourceFile) {
initializeResourceTypesIfNeeded(conf, resourceFile);
return resourceTypes; return resourceTypes;
} }

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
View File

@ -173,7 +173,7 @@ public static Resource clone(Resource res) {
} }
public static Resource addTo(Resource lhs, Resource rhs) { public static Resource addTo(Resource lhs, Resource rhs) {
int maxLength = ResourceUtils.getResourceTypesArray().length; int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) { for (int i = 0; i < maxLength; i++) {
try { try {
ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation rhsValue = rhs.getResourceInformation(i);
@ -197,7 +197,7 @@ public static Resource add(Resource lhs, Resource rhs) {
} }
public static Resource subtractFrom(Resource lhs, Resource rhs) { public static Resource subtractFrom(Resource lhs, Resource rhs) {
int maxLength = ResourceUtils.getResourceTypesArray().length; int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) { for (int i = 0; i < maxLength; i++) {
try { try {
ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation rhsValue = rhs.getResourceInformation(i);
@ -243,7 +243,7 @@ public static Resource negate(Resource resource) {
} }
public static Resource multiplyTo(Resource lhs, double by) { public static Resource multiplyTo(Resource lhs, double by) {
int maxLength = ResourceUtils.getResourceTypesArray().length; int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) { for (int i = 0; i < maxLength; i++) {
try { try {
ResourceInformation lhsValue = lhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i);
@ -266,7 +266,7 @@ public static Resource multiply(Resource lhs, double by) {
*/ */
public static Resource multiplyAndAddTo( public static Resource multiplyAndAddTo(
Resource lhs, Resource rhs, double by) { Resource lhs, Resource rhs, double by) {
int maxLength = ResourceUtils.getResourceTypesArray().length; int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) { for (int i = 0; i < maxLength; i++) {
try { try {
ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation rhsValue = rhs.getResourceInformation(i);
@ -299,7 +299,7 @@ public static Resource multiplyAndNormalizeDown(
public static Resource multiplyAndRoundDown(Resource lhs, double by) { public static Resource multiplyAndRoundDown(Resource lhs, double by) {
Resource out = clone(lhs); Resource out = clone(lhs);
int maxLength = ResourceUtils.getResourceTypesArray().length; int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) { for (int i = 0; i < maxLength; i++) {
try { try {
ResourceInformation lhsValue = lhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i);
@ -408,7 +408,7 @@ public static Resource max(
} }
public static boolean fitsIn(Resource smaller, Resource bigger) { public static boolean fitsIn(Resource smaller, Resource bigger) {
int maxLength = ResourceUtils.getResourceTypesArray().length; int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) { for (int i = 0; i < maxLength; i++) {
try { try {
ResourceInformation rhsValue = bigger.getResourceInformation(i); ResourceInformation rhsValue = bigger.getResourceInformation(i);
@ -436,7 +436,7 @@ public static boolean fitsIn(ResourceCalculator rc, Resource cluster,
public static Resource componentwiseMin(Resource lhs, Resource rhs) { public static Resource componentwiseMin(Resource lhs, Resource rhs) {
Resource ret = createResource(0); Resource ret = createResource(0);
int maxLength = ResourceUtils.getResourceTypesArray().length; int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) { for (int i = 0; i < maxLength; i++) {
try { try {
ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation rhsValue = rhs.getResourceInformation(i);
@ -460,7 +460,7 @@ public static Resource componentwiseMin(Resource lhs, Resource rhs) {
public static Resource componentwiseMax(Resource lhs, Resource rhs) { public static Resource componentwiseMax(Resource lhs, Resource rhs) {
Resource ret = createResource(0); Resource ret = createResource(0);
int maxLength = ResourceUtils.getResourceTypesArray().length; int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) { for (int i = 0; i < maxLength; i++) {
try { try {
ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation rhsValue = rhs.getResourceInformation(i);