YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java

@@ -36,7 +36,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
   private static final String VCORES_URI = "vcores";
 
   public static final ResourceInformation MEMORY_MB =
-      ResourceInformation.newInstance(MEMORY_URI, "M");
+      ResourceInformation.newInstance(MEMORY_URI, "Mi");
   public static final ResourceInformation VCORES =
       ResourceInformation.newInstance(VCORES_URI);
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java

@@ -46,7 +46,8 @@ public static class Converter {
   }
 
   private static final String[] UNITS =
-      {"p", "n", "u", "m", "", "k", "M", "G", "T", "P"};
+      { "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti",
+          "Pi" };
   private static final List<String> SORTED_UNITS = Arrays.asList(UNITS);
   public static final Set<String> KNOWN_UNITS = createKnownUnitsSet();
   private static final Converter PICO =
@@ -65,6 +66,15 @@ public static class Converter {
   private static final Converter PETA =
       new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L);
 
+  private static final Converter KILO_BINARY = new Converter(1024L, 1L);
+  private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 1L);
+  private static final Converter GIGA_BINARY =
+      new Converter(1024L * 1024L * 1024L, 1L);
+  private static final Converter TERA_BINARY =
+      new Converter(1024L * 1024L * 1024L * 1024L, 1L);
+  private static final Converter PETA_BINARY =
+      new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L);
+
   private static Set<String> createKnownUnitsSet() {
     Set<String> ret = new HashSet<>();
     ret.addAll(Arrays.asList(UNITS));
@@ -93,6 +103,16 @@ private static Converter getConverter(String unit) {
       return TERA;
     case "P":
       return PETA;
+    case "Ki":
+      return KILO_BINARY;
+    case "Mi":
+      return MEGA_BINARY;
+    case "Gi":
+      return GIGA_BINARY;
+    case "Ti":
+      return TERA_BINARY;
+    case "Pi":
+      return PETA_BINARY;
     default:
       throw new IllegalArgumentException(
           "Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS);
@@ -112,29 +132,30 @@ public static Long convert(String fromUnit, String toUnit, Long fromValue) {
     if (toUnit == null || fromUnit == null || fromValue == null) {
       throw new IllegalArgumentException("One or more arguments are null");
     }
-    Long tmp;
     String overflowMsg =
         "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
             + "' will result in an overflow of Long";
+    if (fromUnit.equals(toUnit)) {
+      return fromValue;
+    }
     Converter fc = getConverter(fromUnit);
     Converter tc = getConverter(toUnit);
     Long numerator = fc.numerator * tc.denominator;
     Long denominator = fc.denominator * tc.numerator;
+    Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator;
     if (numerator < denominator) {
-      if (!toUnit.equals(fromUnit)) {
+      if (numeratorMultiplierLimit < fromValue) {
-        tmp = Long.MAX_VALUE / numerator;
-        if (tmp < fromValue) {
         throw new IllegalArgumentException(overflowMsg);
       }
-      }
       return (fromValue * numerator) / denominator;
     }
-    tmp = numerator / denominator;
+    if (numeratorMultiplierLimit > fromValue) {
-    if (!toUnit.equals(fromUnit)) {
+      return (numerator * fromValue) / denominator;
+    }
+    Long tmp = numerator / denominator;
     if ((Long.MAX_VALUE / tmp) < fromValue) {
       throw new IllegalArgumentException(overflowMsg);
     }
-    }
     return fromValue * tmp;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java

@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.util;
 
-import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -45,6 +44,7 @@ public void testUnitsConversion() {
     fromUnit = "";
     Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l),
         UnitsConversionUtil.convert(fromUnit, "k", test));
+
     Assert
         .assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)),
             UnitsConversionUtil.convert(fromUnit, "M", test));
@@ -63,6 +63,21 @@ public void testUnitsConversion() {
 
     Assert.assertEquals("mega to giga test failed", Long.valueOf(value),
         UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l)));
+
+    Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value),
+        UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l)));
+
+    Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024),
+        UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value)));
+
+    Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024),
+        UnitsConversionUtil.convert("Ki", "", Long.valueOf(5)));
+
+    Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741),
+        UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024)));
+
+    Assert.assertEquals("M to Mi test failed", Long.valueOf(953),
+        UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000)));
   }
 
   @Test

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java

@@ -95,7 +95,7 @@ public long getMemorySize() {
     initResourcesMap();
     ResourceInformation ri =
         this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
-    return UnitsConversionUtil.convert(ri.getUnits(), "M", ri.getValue());
+    return UnitsConversionUtil.convert(ri.getUnits(), "Mi", ri.getValue());
   }
 
   @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java

@@ -195,7 +195,7 @@ public void testInitializeResourcesMap() throws Exception {
       // we must always have memory and vcores with their fixed units
       Assert.assertTrue(ret.containsKey("memory-mb"));
       ResourceInformation memInfo = ret.get("memory-mb");
-      Assert.assertEquals("M", memInfo.getUnits());
+      Assert.assertEquals("Mi", memInfo.getUnits());
       Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType());
       Assert.assertTrue(ret.containsKey("vcores"));
       ResourceInformation vcoresInfo = ret.get("vcores");