diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FastNumberFormat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FastNumberFormat.java new file mode 100644 index 0000000000..9a6b63f281 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FastNumberFormat.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +/** + * Fast thread-safe version of NumberFormat + */ +public class FastNumberFormat { + + public static StringBuilder format(StringBuilder sb, long value, int minimumDigits) { + if (value < 0) { + sb.append('-'); + value = -value; + } + + long tmp = value; + do { + tmp /= 10; + } while (--minimumDigits > 0 && tmp > 0); + + for (int i = minimumDigits; i > 0; --i) { + sb.append('0'); + } + + sb.append(value); + return sb; + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFastNumberFormat.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFastNumberFormat.java new file mode 100644 index 0000000000..c8935dde3a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFastNumberFormat.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.junit.Assert; +import org.junit.Test; + +import java.text.NumberFormat; + +/** + * Test for FastNumberFormat + */ +public class TestFastNumberFormat { + private final int MIN_DIGITS = 6; + + @Test(timeout = 1000) + public void testLongWithPadding() throws Exception { + NumberFormat numberFormat = NumberFormat.getInstance(); + numberFormat.setGroupingUsed(false); + numberFormat.setMinimumIntegerDigits(6); + long[] testLongs = {1, 23, 456, 7890, 12345, 678901, 2345689, 0, -0, -1, + -23, -456, -7890, -12345, -678901, -2345689}; + for (long l: testLongs) { + StringBuilder sb = new StringBuilder(); + FastNumberFormat.format(sb, l, MIN_DIGITS); + String fastNumberStr = sb.toString(); + Assert.assertEquals("Number formats should be equal", + numberFormat.format(l), fastNumberStr); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java index 5f3a68ebe1..dbaacd9176 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java @@ -18,18 +18,13 @@ package org.apache.hadoop.yarn.api.records; -import java.text.NumberFormat; -import java.util.Iterator; -import java.util.NoSuchElementException; - import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.FastNumberFormat; import org.apache.hadoop.yarn.util.Records; -import com.google.common.base.Splitter; - /** *

ApplicationAttemptId denotes the particular attempt * of an ApplicationMaster for a given {@link ApplicationId}.

@@ -42,12 +37,14 @@ import com.google.common.base.Splitter; @Stable public abstract class ApplicationAttemptId implements Comparable { - private static Splitter _spliter = Splitter.on('_').trimResults(); @Private @Unstable public static final String appAttemptIdStrPrefix = "appattempt"; + private static final String APP_ATTEMPT_ID_PREFIX = appAttemptIdStrPrefix + + '_'; + @Public @Unstable public static ApplicationAttemptId newInstance(ApplicationId appId, @@ -84,16 +81,8 @@ public abstract class ApplicationAttemptId implements @Unstable protected abstract void setAttemptId(int attemptId); - static final ThreadLocal attemptIdFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(6); - return fmt; - } - }; + private static final int ATTEMPT_ID_MIN_DIGITS = 6; + private static final int APP_ID_MIN_DIGITS = 4; @Override public int hashCode() { @@ -135,12 +124,14 @@ public abstract class ApplicationAttemptId implements @Override public String toString() { - StringBuilder sb = new StringBuilder(appAttemptIdStrPrefix); - sb.append("_"); - sb.append(this.getApplicationId().getClusterTimestamp()).append("_"); - sb.append(ApplicationId.appIdFormat.get().format( - this.getApplicationId().getId())); - sb.append("_").append(attemptIdFormat.get().format(getAttemptId())); + StringBuilder sb = new StringBuilder(64); + sb.append(APP_ATTEMPT_ID_PREFIX); + ApplicationId appId = getApplicationId(); + sb.append(appId.getClusterTimestamp()); + sb.append('_'); + FastNumberFormat.format(sb, appId.getId(), APP_ID_MIN_DIGITS); + sb.append('_'); + FastNumberFormat.format(sb, getAttemptId(), ATTEMPT_ID_MIN_DIGITS); return sb.toString(); } @@ -148,29 +139,33 @@ public abstract class ApplicationAttemptId implements @Public @Stable - public static ApplicationAttemptId fromString(String applicationAttemptIdStr) { - Iterator it = _spliter.split(applicationAttemptIdStr).iterator(); - if (!it.next().equals(appAttemptIdStrPrefix)) { + public static ApplicationAttemptId fromString(String appAttemptIdStr) { + if (!appAttemptIdStr.startsWith(APP_ATTEMPT_ID_PREFIX)) { throw new IllegalArgumentException("Invalid AppAttemptId prefix: " - + applicationAttemptIdStr); + + appAttemptIdStr); } try { - return toApplicationAttemptId(it); + int pos1 = APP_ATTEMPT_ID_PREFIX.length() - 1; + int pos2 = appAttemptIdStr.indexOf('_', pos1 + 1); + if (pos2 < 0) { + throw new IllegalArgumentException("Invalid AppAttemptId: " + + appAttemptIdStr); + } + long rmId = Long.parseLong(appAttemptIdStr.substring(pos1 + 1, pos2)); + int pos3 = appAttemptIdStr.indexOf('_', pos2 + 1); + if (pos3 < 0) { + throw new IllegalArgumentException("Invalid AppAttemptId: " + + appAttemptIdStr); + } + int appId = Integer.parseInt(appAttemptIdStr.substring(pos2 + 1, pos3)); + ApplicationId applicationId = ApplicationId.newInstance(rmId, appId); + int attemptId = Integer.parseInt(appAttemptIdStr.substring(pos3 + 1)); + ApplicationAttemptId applicationAttemptId = + ApplicationAttemptId.newInstance(applicationId, attemptId); + return applicationAttemptId; } catch (NumberFormatException n) { throw new IllegalArgumentException("Invalid AppAttemptId: " - + applicationAttemptIdStr, n); - } catch (NoSuchElementException e) { - throw new IllegalArgumentException("Invalid AppAttemptId: " - + applicationAttemptIdStr, e); + + appAttemptIdStr, n); } } - - private static ApplicationAttemptId toApplicationAttemptId( - Iterator it) throws NumberFormatException { - ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), - Integer.parseInt(it.next())); - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next())); - return appAttemptId; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java index 03a77ce309..392e0a1d88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java @@ -18,18 +18,13 @@ package org.apache.hadoop.yarn.api.records; -import java.text.NumberFormat; -import java.util.Iterator; -import java.util.NoSuchElementException; - import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.FastNumberFormat; import org.apache.hadoop.yarn.util.Records; -import com.google.common.base.Splitter; - /** *

ApplicationId represents the globally unique * identifier for an application.

@@ -42,12 +37,13 @@ import com.google.common.base.Splitter; @Public @Stable public abstract class ApplicationId implements Comparable { - private static Splitter _spliter = Splitter.on('_').trimResults(); @Private @Unstable public static final String appIdStrPrefix = "application"; + private static final String APPLICATION_ID_PREFIX = appIdStrPrefix + '_'; + @Public @Unstable public static ApplicationId newInstance(long clusterTimestamp, int id) { @@ -87,16 +83,7 @@ public abstract class ApplicationId implements Comparable { protected abstract void build(); - static final ThreadLocal appIdFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(4); - return fmt; - } - }; + private static final int APP_ID_MIN_DIGITS = 4; @Override public int compareTo(ApplicationId other) { @@ -110,37 +97,38 @@ public abstract class ApplicationId implements Comparable { @Override public String toString() { - return appIdStrPrefix + "_" + this.getClusterTimestamp() + "_" + appIdFormat - .get().format(getId()); - } - - private static ApplicationId toApplicationId( - Iterator it) throws NumberFormatException { - ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), - Integer.parseInt(it.next())); - return appId; + StringBuilder sb = new StringBuilder(64); + sb.append(APPLICATION_ID_PREFIX); + sb.append(getClusterTimestamp()); + sb.append('_'); + FastNumberFormat.format(sb, getId(), APP_ID_MIN_DIGITS); + return sb.toString(); } @Public @Stable public static ApplicationId fromString(String appIdStr) { - Iterator it = _spliter.split((appIdStr)).iterator(); - if (!it.next().equals(appIdStrPrefix)) { + if (!appIdStr.startsWith(APPLICATION_ID_PREFIX)) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + appIdStrPrefix); } try { - return toApplicationId(it); + int pos1 = APPLICATION_ID_PREFIX.length() - 1; + int pos2 = appIdStr.indexOf('_', pos1 + 1); + if (pos2 < 0) { + throw new IllegalArgumentException("Invalid ApplicationId: " + + appIdStr); + } + long rmId = Long.parseLong(appIdStr.substring(pos1 + 1, pos2)); + int appId = Integer.parseInt(appIdStr.substring(pos2 + 1)); + ApplicationId applicationId = ApplicationId.newInstance(rmId, appId); + return applicationId; } catch (NumberFormatException n) { throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr, n); - } catch (NoSuchElementException e) { - throw new IllegalArgumentException("Invalid ApplicationId: " - + appIdStr, e); } } - @Override public int hashCode() { // Generated by eclipse. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java index feddeca9e7..e506dc7c84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java @@ -18,16 +18,11 @@ package org.apache.hadoop.yarn.api.records; -import com.google.common.base.Splitter; - -import java.text.NumberFormat; -import java.util.Iterator; -import java.util.NoSuchElementException; - import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.FastNumberFormat; import org.apache.hadoop.yarn.util.Records; /** @@ -38,8 +33,7 @@ import org.apache.hadoop.yarn.util.Records; @Stable public abstract class ContainerId implements Comparable{ public static final long CONTAINER_ID_BITMASK = 0xffffffffffL; - private static final Splitter _SPLITTER = Splitter.on('_').trimResults(); - private static final String CONTAINER_PREFIX = "container"; + private static final String CONTAINER_PREFIX = "container_"; private static final String EPOCH_PREFIX = "e"; @Public @@ -115,29 +109,13 @@ public abstract class ContainerId implements Comparable{ protected abstract void setContainerId(long id); - // TODO: fail the app submission if attempts are more than 10 or something - private static final ThreadLocal appAttemptIdAndEpochFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(2); - return fmt; - } - }; - // TODO: Why thread local? - // ^ NumberFormat instances are not threadsafe - private static final ThreadLocal containerIdFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(6); - return fmt; - } - }; + private static final int APP_ID_MIN_DIGITS = 4; + + private static final int ATTEMPT_ID_MIN_DIGITS = 2; + + private static final int EPOCH_MIN_DIGITS = 2; + + private static final int CONTAINER_ID_MIN_DIGITS = 6; @Override public int hashCode() { @@ -185,72 +163,85 @@ public abstract class ContainerId implements Comparable{ */ @Override public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(CONTAINER_PREFIX + "_"); + StringBuilder sb = new StringBuilder(64); + sb.append(CONTAINER_PREFIX); long epoch = getContainerId() >> 40; if (epoch > 0) { - sb.append(EPOCH_PREFIX) - .append(appAttemptIdAndEpochFormat.get().format(epoch)).append("_");; + sb.append(EPOCH_PREFIX); + FastNumberFormat.format(sb, epoch, EPOCH_MIN_DIGITS); + sb.append('_'); } ApplicationId appId = getApplicationAttemptId().getApplicationId(); - sb.append(appId.getClusterTimestamp()).append("_"); - sb.append(ApplicationId.appIdFormat.get().format(appId.getId())) - .append("_"); - sb.append( - appAttemptIdAndEpochFormat.get().format( - getApplicationAttemptId().getAttemptId())).append("_"); - sb.append(containerIdFormat.get() - .format(CONTAINER_ID_BITMASK & getContainerId())); + sb.append(appId.getClusterTimestamp()); + sb.append('_'); + FastNumberFormat.format(sb, appId.getId(), APP_ID_MIN_DIGITS); + sb.append('_'); + FastNumberFormat.format(sb, getApplicationAttemptId().getAttemptId(), + ATTEMPT_ID_MIN_DIGITS); + sb.append('_'); + FastNumberFormat.format(sb, CONTAINER_ID_BITMASK & getContainerId(), + CONTAINER_ID_MIN_DIGITS); return sb.toString(); } @Public @Stable public static ContainerId fromString(String containerIdStr) { - Iterator it = _SPLITTER.split(containerIdStr).iterator(); - if (!it.next().equals(CONTAINER_PREFIX)) { + if (!containerIdStr.startsWith(CONTAINER_PREFIX)) { throw new IllegalArgumentException("Invalid ContainerId prefix: " + containerIdStr); } try { - String epochOrClusterTimestampStr = it.next(); + int pos1 = CONTAINER_PREFIX.length() - 1; + long epoch = 0; - ApplicationAttemptId appAttemptID = null; - if (epochOrClusterTimestampStr.startsWith(EPOCH_PREFIX)) { - String epochStr = epochOrClusterTimestampStr; - epoch = Integer.parseInt(epochStr.substring(EPOCH_PREFIX.length())); - appAttemptID = toApplicationAttemptId(it); - } else { - String clusterTimestampStr = epochOrClusterTimestampStr; - long clusterTimestamp = Long.parseLong(clusterTimestampStr); - appAttemptID = toApplicationAttemptId(clusterTimestamp, it); + if (containerIdStr.regionMatches(pos1 + 1, EPOCH_PREFIX, 0, + EPOCH_PREFIX.length())) { + int pos2 = containerIdStr.indexOf('_', pos1 + 1); + if (pos2 < 0) { + throw new IllegalArgumentException("Invalid ContainerId: " + + containerIdStr); + } + String epochStr = containerIdStr.substring( + pos1 + 1 + EPOCH_PREFIX.length(), pos2); + epoch = Integer.parseInt(epochStr); + // rewind the current position + pos1 = pos2; } - long id = Long.parseLong(it.next()); + int pos2 = containerIdStr.indexOf('_', pos1 + 1); + if (pos2 < 0) { + throw new IllegalArgumentException("Invalid ContainerId: " + + containerIdStr); + } + long clusterTimestamp = Long.parseLong( + containerIdStr.substring(pos1 + 1, pos2)); + + int pos3 = containerIdStr.indexOf('_', pos2 + 1); + if (pos3 < 0) { + throw new IllegalArgumentException("Invalid ContainerId: " + + containerIdStr); + } + int appId = Integer.parseInt(containerIdStr.substring(pos2 + 1, pos3)); + ApplicationId applicationId = ApplicationId.newInstance(clusterTimestamp, + appId); + int pos4 = containerIdStr.indexOf('_', pos3 + 1); + if (pos4 < 0) { + throw new IllegalArgumentException("Invalid ContainerId: " + + containerIdStr); + } + int attemptId = Integer.parseInt( + containerIdStr.substring(pos3 + 1, pos4)); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(applicationId, attemptId); + long id = Long.parseLong(containerIdStr.substring(pos4 + 1)); long cid = (epoch << 40) | id; - ContainerId containerId = ContainerId.newContainerId(appAttemptID, cid); + ContainerId containerId = ContainerId.newContainerId(appAttemptId, cid); return containerId; } catch (NumberFormatException n) { throw new IllegalArgumentException("Invalid ContainerId: " + containerIdStr, n); - } catch (NoSuchElementException e) { - throw new IllegalArgumentException("Invalid ContainerId: " - + containerIdStr, e); } } - private static ApplicationAttemptId toApplicationAttemptId( - Iterator it) throws NumberFormatException { - return toApplicationAttemptId(Long.parseLong(it.next()), it); - } - - private static ApplicationAttemptId toApplicationAttemptId( - long clusterTimestamp, Iterator it) throws NumberFormatException { - ApplicationId appId = ApplicationId.newInstance(clusterTimestamp, - Integer.parseInt(it.next())); - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next())); - return appAttemptId; - } - protected abstract void build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java index 71474b14b8..a0c209d570 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java @@ -19,11 +19,11 @@ package org.apache.hadoop.yarn.api.records; import java.io.IOException; -import java.text.NumberFormat; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.FastNumberFormat; import org.apache.hadoop.yarn.util.Records; /** @@ -89,16 +89,7 @@ public abstract class ReservationId implements Comparable { protected abstract void build(); - static final ThreadLocal reservIdFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(4); - return fmt; - } - }; + private static final int RESERVATION_ID_MIN_DIGITS = 4; @Override public int compareTo(ReservationId other) { @@ -112,8 +103,12 @@ public abstract class ReservationId implements Comparable { @Override public String toString() { - return reserveIdStrPrefix + this.getClusterTimestamp() + "_" - + reservIdFormat.get().format(getId()); + StringBuilder sb = new StringBuilder(64); + sb.append(reserveIdStrPrefix); + sb.append(getClusterTimestamp()); + sb.append('_'); + FastNumberFormat.format(sb, getId(), RESERVATION_ID_MIN_DIGITS); + return sb.toString(); } /**