From 5a38ed2f2274ccfad100cc89c729d470063e270d Mon Sep 17 00:00:00 2001 From: Fei Hui Date: Sun, 4 Oct 2020 01:02:28 +0800 Subject: [PATCH] HADOOP-17276. Extend CallerContext to make it include many items (#2327) Cherry-picked from d0d10f7e by Owen O'Malley --- .../fs/CommonConfigurationKeysPublic.java | 3 + .../org/apache/hadoop/ipc/CallerContext.java | 105 +++++++++++++++++- .../src/main/resources/core-default.xml | 10 ++ .../apache/hadoop/ipc/TestCallerContext.java | 53 +++++++++ .../metrics/TimelineServiceV1Publisher.java | 2 +- .../metrics/TimelineServiceV2Publisher.java | 2 +- .../impl/pb/ApplicationStateDataPBImpl.java | 7 +- .../resourcemanager/TestRMAuditLogger.java | 4 +- 8 files changed, 176 insertions(+), 10 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 1fa84c7c15..d769b284c6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -371,6 +371,9 @@ public class CommonConfigurationKeysPublic { "hadoop.caller.context.signature.max.size"; public static final int HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT = 40; + public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_KEY = + "hadoop.caller.context.separator"; + public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT = ","; /** * @see diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java index dd14ba133e..e15340c618 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java @@ -21,10 +21,17 @@ import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY; /** * A class defining the caller context for auditing coarse granularity @@ -54,8 +61,8 @@ public final class CallerContext { private final byte[] signature; private CallerContext(Builder builder) { - this.context = builder.context; - this.signature = builder.signature; + this.context = builder.getContext(); + this.signature = builder.getSignature(); } public String getContext() { @@ -109,11 +116,53 @@ public String toString() { /** The caller context builder. */ public static final class Builder { - private final String context; + private static final String KEY_VALUE_SEPARATOR = ":"; + /** + * The illegal separators include '\t', '\n', '='. + * User should not set illegal separator. + */ + private static final Set ILLEGAL_SEPARATORS = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("\t", "\n", "="))); + private final String fieldSeparator; + private final StringBuilder sb = new StringBuilder(); private byte[] signature; public Builder(String context) { - this.context = context; + this(context, new Configuration()); + } + + public Builder(String context, Configuration conf) { + if (isValid(context)) { + sb.append(context); + } + fieldSeparator = conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, + HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT); + checkFieldSeparator(fieldSeparator); + } + + /** + * Check whether the separator is legal. + * The illegal separators include '\t', '\n', '='. + * Throw IllegalArgumentException if the separator is Illegal. + * @param separator the separator of fields. + */ + private void checkFieldSeparator(String separator) { + if (ILLEGAL_SEPARATORS.contains(separator)) { + throw new IllegalArgumentException("Illegal field separator: " + + separator); + } + } + + /** + * Whether the field is valid. + * The field should not contain '\t', '\n', '='. + * Because the context could be written to audit log. + * @param field one of the fields in context. + * @return true if the field is not null or empty. + */ + private boolean isValid(String field) { + return field != null && field.length() > 0; } public Builder setSignature(byte[] signature) { @@ -123,6 +172,54 @@ public Builder setSignature(byte[] signature) { return this; } + /** + * Get the context. + * For example, the context is "key1:value1,key2:value2". + * @return the valid context or null. + */ + public String getContext() { + return sb.length() > 0 ? sb.toString() : null; + } + + /** + * Get the signature. + * @return the signature. + */ + public byte[] getSignature() { + return signature; + } + + /** + * Append new field to the context. + * @param field one of fields to append. + * @return the builder. + */ + public Builder append(String field) { + if (isValid(field)) { + if (sb.length() > 0) { + sb.append(fieldSeparator); + } + sb.append(field); + } + return this; + } + + /** + * Append new field which contains key and value to the context. + * @param key the key of field. + * @param value the value of field. + * @return the builder. + */ + public Builder append(String key, String value) { + if (isValid(key) && isValid(value)) { + if (sb.length() > 0) { + sb.append(fieldSeparator); + } + sb.append(key).append(KEY_VALUE_SEPARATOR).append(value); + } + return this; + } + public CallerContext build() { return new CallerContext(this); } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index f5978a3de8..8a59f3c178 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -3716,6 +3716,16 @@ The switch to turn S3A auditing on or off. in audit logs. + + hadoop.caller.context.separator + , + + The separator is for context which maybe contain many fields. For example, + if the separator is ',', and there are two key/value fields in context, + in which case the context string is "key1:value1,key2:value2". The + separator should not contain '\t', '\n', '='. + + seq.io.sort.mb diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java new file mode 100644 index 0000000000..fc1057b9f7 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ipc; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY; + +public class TestCallerContext { + @Test + public void testBuilderAppend() { + Configuration conf = new Configuration(); + conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "$"); + CallerContext.Builder builder = new CallerContext.Builder(null, conf); + CallerContext context = builder.append("context1") + .append("context2").append("key3", "value3").build(); + Assert.assertEquals(true, + context.getContext().contains("$")); + String[] items = context.getContext().split("\\$"); + Assert.assertEquals(3, items.length); + Assert.assertEquals("key3:value3", items[2]); + + builder.append("$$"); + Assert.assertEquals("context1$context2$key3:value3$$$", + builder.build().getContext()); + } + + @Test(expected = IllegalArgumentException.class) + public void testNewBuilder() { + Configuration conf = new Configuration(); + // Set illegal separator. + conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "\t"); + CallerContext.Builder builder = new CallerContext.Builder(null, conf); + builder.build(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index 1d89f1e522..f1b80a946a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -176,7 +176,7 @@ public void appCreated(RMApp app, long createdTime) { entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, app.getAppNodeLabelExpression()); if (app.getCallerContext() != null) { - if (app.getCallerContext().getContext() != null) { + if (app.getCallerContext().isContextValid()) { entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT, app.getCallerContext().getContext()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index d359173027..8047acb34b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -127,7 +127,7 @@ public void appCreated(RMApp app, long createdTime) { ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, app.getAppNodeLabelExpression()); if (app.getCallerContext() != null) { - if (app.getCallerContext().getContext() != null) { + if (app.getCallerContext().isContextValid()) { entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT, app.getCallerContext().getContext()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java index 5dcb70bdd8..c0f7bb0262 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java @@ -274,14 +274,17 @@ public void setCallerContext(CallerContext callerContext) { RpcHeaderProtos.RPCCallerContextProto.Builder b = RpcHeaderProtos.RPCCallerContextProto .newBuilder(); - if (callerContext.getContext() != null) { + if (callerContext.isContextValid()) { b.setContext(callerContext.getContext()); } if (callerContext.getSignature() != null) { b.setSignature(ByteString.copyFrom(callerContext.getSignature())); } - builder.setCallerContext(b); + if(callerContext.isContextValid() + || callerContext.getSignature() != null) { + builder.setCallerContext(b); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index e8a532d1ba..c78a7788c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -177,7 +177,7 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, expLog.append("\tRESOURCE="); } if (callerContext != null) { - if (callerContext.getContext() != null) { + if (callerContext.isContextValid()) { expLog.append("\tCALLERCONTEXT=context"); } if (callerContext.getSignature() != null) { @@ -328,7 +328,7 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, expLog.append("\tRESOURCE="); } if (callerContext != null) { - if (callerContext.getContext() != null) { + if (callerContext.isContextValid()) { expLog.append("\tCALLERCONTEXT=context"); } if (callerContext.getSignature() != null) {