HADOOP-7910. Add Configuration.getLongBytes to handle human readable byte size values. (Sho Shimauchi via harsh)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1225489 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -12,6 +12,8 @@ Trunk (unreleased changes)
     HADOOP-7875. Add helper class to unwrap protobuf ServiceException.
     (suresh)
 
+    HADOOP-7910. Add Configuration.getLongBytes to handle human readable byte size values. (Sho Shimauchi via harsh)
+
   IMPROVEMENTS
 
     HADOOP-7595. Upgrade dependency to Avro 1.5.3. (Alejandro Abdelnur via atm)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

@@ -737,6 +737,27 @@ public long getLong(String name, long defaultValue) {
     return Long.parseLong(valueString);
   }
 
+  /**
+   * Get the value of the <code>name</code> property as a <code>long</code> or
+   * human readable format. If no such property exists, the provided default
+   * value is returned, or if the specified value is not a valid
+   * <code>long</code> or human readable format, then an error is thrown. You
+   * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+   * t(tera), p(peta), e(exa)
+   *
+   * @param name property name.
+   * @param defaultValue default value.
+   * @throws NumberFormatException when the value is invalid
+   * @return property value as a <code>long</code>,
+   *         or <code>defaultValue</code>.
+   */
+  public long getLongBytes(String name, long defaultValue) {
+    String valueString = getTrimmed(name);
+    if (valueString == null)
+      return defaultValue;
+    return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
+  }
+
   private String getHexDigits(String value) {
     boolean negative = false;
     String str = value;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -661,7 +661,14 @@ public static long string2long(String s) {
       if (Character.isDigit(lastchar))
         return Long.parseLong(s);
       else {
-        long prefix = TraditionalBinaryPrefix.valueOf(lastchar).value;
+        long prefix;
+        try {
+          prefix = TraditionalBinaryPrefix.valueOf(lastchar).value;
+        } catch (IllegalArgumentException e) {
+          throw new IllegalArgumentException("Invalid size prefix '" + lastchar
+              + "' in '" + s
+              + "'. Allowed prefixes are k, m, g, t, p, e(case insensitive)");
+        }
         long num = Long.parseLong(s.substring(0, lastpos));
         if (num > (Long.MAX_VALUE/prefix) || num < (Long.MIN_VALUE/prefix)) {
           throw new IllegalArgumentException(s + " does not fit in a Long");

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java

@@ -405,12 +405,16 @@ public void testIntegerValues() throws IOException{
     conf.addResource(fileResource);
     assertEquals(20, conf.getInt("test.int1", 0));
     assertEquals(20, conf.getLong("test.int1", 0));
+    assertEquals(20, conf.getLongBytes("test.int1", 0));
     assertEquals(20, conf.getInt("test.int2", 0));
     assertEquals(20, conf.getLong("test.int2", 0));
+    assertEquals(20, conf.getLongBytes("test.int2", 0));
     assertEquals(-20, conf.getInt("test.int3", 0));
     assertEquals(-20, conf.getLong("test.int3", 0));
+    assertEquals(-20, conf.getLongBytes("test.int3", 0));
     assertEquals(-20, conf.getInt("test.int4", 0));
     assertEquals(-20, conf.getLong("test.int4", 0));
+    assertEquals(-20, conf.getLongBytes("test.int4", 0));
     try {
       conf.getInt("test.int5", 0);
       fail("Property had invalid int value, but was read successfully.");
@@ -419,6 +423,26 @@ public void testIntegerValues() throws IOException{
     }
   }
   
+  public void testHumanReadableValues() throws IOException {
+    out = new BufferedWriter(new FileWriter(CONFIG));
+    startConfig();
+    appendProperty("test.humanReadableValue1", "1m");
+    appendProperty("test.humanReadableValue2", "1M");
+    appendProperty("test.humanReadableValue5", "1MBCDE");
+
+    endConfig();
+    Path fileResource = new Path(CONFIG);
+    conf.addResource(fileResource);
+    assertEquals(1048576, conf.getLongBytes("test.humanReadableValue1", 0));
+    assertEquals(1048576, conf.getLongBytes("test.humanReadableValue2", 0));
+    try {
+      conf.getLongBytes("test.humanReadableValue5", 0);
+      fail("Property had invalid human readable value, but was read successfully.");
+    } catch (NumberFormatException e) {
+      // pass
+    }
+  }
+
   public void testBooleanValues() throws IOException {
     out=new BufferedWriter(new FileWriter(CONFIG));
     startConfig();

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java

@@ -143,8 +143,62 @@ public void testTraditionalBinaryPrefix() throws Exception {
     }
     
     assertEquals(0L, StringUtils.TraditionalBinaryPrefix.string2long("0"));
-    assertEquals(-1259520L, StringUtils.TraditionalBinaryPrefix.string2long("-1230k"));
+    assertEquals(1024L, StringUtils.TraditionalBinaryPrefix.string2long("1k"));
-    assertEquals(956703965184L, StringUtils.TraditionalBinaryPrefix.string2long("891g"));
+    assertEquals(-1024L, StringUtils.TraditionalBinaryPrefix.string2long("-1k"));
+    assertEquals(1259520L,
+        StringUtils.TraditionalBinaryPrefix.string2long("1230K"));
+    assertEquals(-1259520L,
+        StringUtils.TraditionalBinaryPrefix.string2long("-1230K"));
+    assertEquals(104857600L,
+        StringUtils.TraditionalBinaryPrefix.string2long("100m"));
+    assertEquals(-104857600L,
+        StringUtils.TraditionalBinaryPrefix.string2long("-100M"));
+    assertEquals(956703965184L,
+        StringUtils.TraditionalBinaryPrefix.string2long("891g"));
+    assertEquals(-956703965184L,
+        StringUtils.TraditionalBinaryPrefix.string2long("-891G"));
+    assertEquals(501377302265856L,
+        StringUtils.TraditionalBinaryPrefix.string2long("456t"));
+    assertEquals(-501377302265856L,
+        StringUtils.TraditionalBinaryPrefix.string2long("-456T"));
+    assertEquals(11258999068426240L,
+        StringUtils.TraditionalBinaryPrefix.string2long("10p"));
+    assertEquals(-11258999068426240L,
+        StringUtils.TraditionalBinaryPrefix.string2long("-10P"));
+    assertEquals(1152921504606846976L,
+        StringUtils.TraditionalBinaryPrefix.string2long("1e"));
+    assertEquals(-1152921504606846976L,
+        StringUtils.TraditionalBinaryPrefix.string2long("-1E"));
+
+    String tooLargeNumStr = "10e";
+    try {
+      StringUtils.TraditionalBinaryPrefix.string2long(tooLargeNumStr);
+      fail("Test passed for a number " + tooLargeNumStr + " too large");
+    } catch (IllegalArgumentException e) {
+      assertEquals(tooLargeNumStr + " does not fit in a Long", e.getMessage());
+    }
+
+    String tooSmallNumStr = "-10e";
+    try {
+      StringUtils.TraditionalBinaryPrefix.string2long(tooSmallNumStr);
+      fail("Test passed for a number " + tooSmallNumStr + " too small");
+    } catch (IllegalArgumentException e) {
+      assertEquals(tooSmallNumStr + " does not fit in a Long", e.getMessage());
+    }
+
+    String invalidFormatNumStr = "10kb";
+    char invalidPrefix = 'b';
+    try {
+      StringUtils.TraditionalBinaryPrefix.string2long(invalidFormatNumStr);
+      fail("Test passed for a number " + invalidFormatNumStr
+          + " has invalid format");
+    } catch (IllegalArgumentException e) {
+      assertEquals("Invalid size prefix '" + invalidPrefix + "' in '"
+          + invalidFormatNumStr
+          + "'. Allowed prefixes are k, m, g, t, p, e(case insensitive)",
+          e.getMessage());
+    }
+
   }
 
   @Test