From cc23514abacb4d6589e731cc5ce5d8e6f19c955d Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Wed, 30 Aug 2017 17:26:13 -0700 Subject: [PATCH] YARN-7115. Move BoundedAppender to org.hadoop.yarn.util pacakge (Contributed by Jian He via Daniel Templeton) --- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 14 +- .../hadoop/yarn/util/BoundedAppender.java | 142 ++++++++++++++++++ .../yarn/util}/TestBoundedAppender.java | 3 +- .../rmapp/attempt/RMAppAttemptImpl.java | 114 +------------- .../TestRMAppAttemptImplDiagnostics.java | 3 +- 5 files changed, 156 insertions(+), 120 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java rename hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt => hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util}/TestBoundedAppender.java (95%) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index f17cf8c500..c2a5c67f5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -108,6 +108,15 @@ org.apache.hadoop hadoop-annotations + + + junit + junit + test + org.mockito mockito-all @@ -128,11 +137,6 @@ com.google.protobuf protobuf-java - - junit - junit - test - org.bouncycastle bcprov-jdk16 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java new file mode 100644 index 0000000000..917d696598 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * A {@link CharSequence} appender that considers its {@link #limit} as upper + * bound. + *

+ * When {@link #limit} would be reached on append, past messages will be + * truncated from head, and a header telling the user about truncation will be + * prepended, with ellipses in between header and messages. + *

+ * Note that header and ellipses are not counted against {@link #limit}. + *

+ * An example: + * + * 

+ * {@code
+ *   // At the beginning it's an empty string
+ *   final Appendable shortAppender = new BoundedAppender(80);
+ *   // The whole message fits into limit
+ *   shortAppender.append(
+ *       "message1 this is a very long message but fitting into limit\n");
+ *   // The first message is truncated, the second not
+ *   shortAppender.append("message2 this is shorter than the previous one\n");
+ *   // The first message is deleted, the second truncated, the third
+ *   // preserved
+ *   shortAppender.append("message3 this is even shorter message, maybe.\n");
+ *   // The first two are deleted, the third one truncated, the last preserved
+ *   shortAppender.append("message4 the shortest one, yet the greatest :)");
+ *   // Current contents are like this:
+ *   // Diagnostic messages truncated, showing last 80 chars out of 199:
+ *   // ...s is even shorter message, maybe.
+ *   // message4 the shortest one, yet the greatest :)
+ * }
+ *
+ *

+ * Note that null values are {@link #append(CharSequence) append}ed + * just like in {@link StringBuilder#append(CharSequence) original + * implementation}. + *

+ * Note that this class is not thread safe. + */ + +@InterfaceAudience.Public +@InterfaceStability.Unstable +@VisibleForTesting +public class BoundedAppender { + @VisibleForTesting + public static final String TRUNCATED_MESSAGES_TEMPLATE = + "Diagnostic messages truncated, showing last " + + "%d chars out of %d:%n...%s"; + + private final int limit; + private final StringBuilder messages = new StringBuilder(); + private int totalCharacterCount = 0; + + public BoundedAppender(final int limit) { + Preconditions.checkArgument(limit > 0, "limit should be positive"); + + this.limit = limit; + } + + /** + * Append a {@link CharSequence} considering {@link #limit}, truncating + * from the head of {@code csq} or {@link #messages} when necessary. + * + * @param csq the {@link CharSequence} to append + * @return this + */ + public BoundedAppender append(final CharSequence csq) { + appendAndCount(csq); + checkAndCut(); + + return this; + } + + private void appendAndCount(final CharSequence csq) { + final int before = messages.length(); + messages.append(csq); + final int after = messages.length(); + totalCharacterCount += after - before; + } + + private void checkAndCut() { + if (messages.length() > limit) { + final int newStart = messages.length() - limit; + messages.delete(0, newStart); + } + } + + /** + * Get current length of messages considering truncates + * without header and ellipses. + * + * @return current length + */ + public int length() { + return messages.length(); + } + + public int getLimit() { + return limit; + } + + /** + * Get a string representation of the actual contents, displaying also a + * header and ellipses when there was a truncate. + * + * @return String representation of the {@link #messages} + */ + @Override + public String toString() { + if (messages.length() < totalCharacterCount) { + return String.format(TRUNCATED_MESSAGES_TEMPLATE, messages.length(), + totalCharacterCount, messages.toString()); + } + + return messages.toString(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java similarity index 95% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java index 9cb1e0404f..2b9cfce25f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java @@ -16,14 +16,13 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; +package org.apache.hadoop.yarn.util; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import static org.junit.Assert.assertEquals; -import static org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl.BoundedAppender; /** * Test class for {@link BoundedAppender}. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index d748860d87..65412df808 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -38,7 +38,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; import javax.crypto.SecretKey; -import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -110,6 +109,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition; import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.BoundedAppender; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -1326,7 +1326,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { // AFTER the initial saving on app-attempt-start // These fields can be visible from outside only after they are saved in // StateStore - BoundedAppender diags = new BoundedAppender(diagnostics.limit); + BoundedAppender diags = new BoundedAppender(diagnostics.getLimit()); // don't leave the tracking URL pointing to a non-existent AM if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, @@ -2295,114 +2295,4 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { return Collections.EMPTY_SET; } - /** - * A {@link CharSequence} appender that considers its {@link #limit} as upper - * bound. - *

- * When {@link #limit} would be reached on append, past messages will be - * truncated from head, and a header telling the user about truncation will be - * prepended, with ellipses in between header and messages. - *

- * Note that header and ellipses are not counted against {@link #limit}. - *

- * An example: - * - * 

-   * {@code
-   *   // At the beginning it's an empty string
-   *   final Appendable shortAppender = new BoundedAppender(80);
-   *   // The whole message fits into limit
-   *   shortAppender.append(
-   *       "message1 this is a very long message but fitting into limit\n");
-   *   // The first message is truncated, the second not
-   *   shortAppender.append("message2 this is shorter than the previous one\n");
-   *   // The first message is deleted, the second truncated, the third
-   *   // preserved
-   *   shortAppender.append("message3 this is even shorter message, maybe.\n");
-   *   // The first two are deleted, the third one truncated, the last preserved
-   *   shortAppender.append("message4 the shortest one, yet the greatest :)");
-   *   // Current contents are like this:
-   *   // Diagnostic messages truncated, showing last 80 chars out of 199:
-   *   // ...s is even shorter message, maybe.
-   *   // message4 the shortest one, yet the greatest :)
-   * }
-   *
- *

- * Note that null values are {@link #append(CharSequence) append}ed - * just like in {@link StringBuilder#append(CharSequence) original - * implementation}. - *

- * Note that this class is not thread safe. - */ - @VisibleForTesting - static class BoundedAppender { - @VisibleForTesting - static final String TRUNCATED_MESSAGES_TEMPLATE = - "Diagnostic messages truncated, showing last " - + "%d chars out of %d:%n...%s"; - - private final int limit; - private final StringBuilder messages = new StringBuilder(); - private int totalCharacterCount = 0; - - BoundedAppender(final int limit) { - Preconditions.checkArgument(limit > 0, "limit should be positive"); - - this.limit = limit; - } - - /** - * Append a {@link CharSequence} considering {@link #limit}, truncating - * from the head of {@code csq} or {@link #messages} when necessary. - * - * @param csq the {@link CharSequence} to append - * @return this - */ - BoundedAppender append(final CharSequence csq) { - appendAndCount(csq); - checkAndCut(); - - return this; - } - - private void appendAndCount(final CharSequence csq) { - final int before = messages.length(); - messages.append(csq); - final int after = messages.length(); - totalCharacterCount += after - before; - } - - private void checkAndCut() { - if (messages.length() > limit) { - final int newStart = messages.length() - limit; - messages.delete(0, newStart); - } - } - - /** - * Get current length of messages considering truncates - * without header and ellipses. - * - * @return current length - */ - int length() { - return messages.length(); - } - - /** - * Get a string representation of the actual contents, displaying also a - * header and ellipses when there was a truncate. - * - * @return String representation of the {@link #messages} - */ - @Override - public String toString() { - if (messages.length() < totalCharacterCount) { - return String.format(TRUNCATED_MESSAGES_TEMPLATE, messages.length(), - totalCharacterCount, messages.toString()); - } - - return messages.toString(); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java index 19b5dd9230..295b59fc8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.util.BoundedAppender; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -86,7 +87,7 @@ public class TestRMAppAttemptImplDiagnostics { appAttempt.appendDiagnostics(beyondLimit); final String truncated = String.format( - RMAppAttemptImpl.BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 1024, + BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 1024, 1025, beyondLimit.substring(1)); assertEquals("messages beyond limit should be truncated", truncated,