big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17276. Extend CallerContext to make it include many items (#2327)

Browse Source 
Cherry-picked from d0d10f7e by Owen O'Malley
This commit is contained in:
Fei Hui 2020-10-04 01:02:28 +08:00 committed by Owen O'Malley
parent 743db6e7b4
commit 5a38ed2f22
8 changed files with 176 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
View File

@ -371,6 +371,9 @@ public class CommonConfigurationKeysPublic {
"hadoop.caller.context.signature.max.size";
public static final int HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT =
40;
public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_KEY =
"hadoop.caller.context.separator";
public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT = ",";
/**
* @see

105
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
View File

@ -21,10 +21,17 @@
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
/**
* A class defining the caller context for auditing coarse granularity
@ -54,8 +61,8 @@ public final class CallerContext {
private final byte[] signature;
private CallerContext(Builder builder) {
this.context = builder.context;
this.signature = builder.signature;
this.context = builder.getContext();
this.signature = builder.getSignature();
}
public String getContext() {
@ -109,11 +116,53 @@ public String toString() {
/** The caller context builder. */
public static final class Builder {
private final String context;
private static final String KEY_VALUE_SEPARATOR = ":";
/**
* The illegal separators include '\t', '\n', '='.
* User should not set illegal separator.
*/
private static final Set<String> ILLEGAL_SEPARATORS =
Collections.unmodifiableSet(
new HashSet<>(Arrays.asList("\t", "\n", "=")));
private final String fieldSeparator;
private final StringBuilder sb = new StringBuilder();
private byte[] signature;
public Builder(String context) {
this.context = context;
this(context, new Configuration());
}
public Builder(String context, Configuration conf) {
if (isValid(context)) {
sb.append(context);
}
fieldSeparator = conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
checkFieldSeparator(fieldSeparator);
}
/**
* Check whether the separator is legal.
* The illegal separators include '\t', '\n', '='.
* Throw IllegalArgumentException if the separator is Illegal.
* @param separator the separator of fields.
*/
private void checkFieldSeparator(String separator) {
if (ILLEGAL_SEPARATORS.contains(separator)) {
throw new IllegalArgumentException("Illegal field separator: "
+ separator);
}
}
/**
* Whether the field is valid.
* The field should not contain '\t', '\n', '='.
* Because the context could be written to audit log.
* @param field one of the fields in context.
* @return true if the field is not null or empty.
*/
private boolean isValid(String field) {
return field != null && field.length() > 0;
}
public Builder setSignature(byte[] signature) {
@ -123,6 +172,54 @@ public Builder setSignature(byte[] signature) {
return this;
}
/**
* Get the context.
* For example, the context is "key1:value1,key2:value2".
* @return the valid context or null.
*/
public String getContext() {
return sb.length() > 0 ? sb.toString() : null;
}
/**
* Get the signature.
* @return the signature.
*/
public byte[] getSignature() {
return signature;
}
/**
* Append new field to the context.
* @param field one of fields to append.
* @return the builder.
*/
public Builder append(String field) {
if (isValid(field)) {
if (sb.length() > 0) {
sb.append(fieldSeparator);
}
sb.append(field);
}
return this;
}
/**
* Append new field which contains key and value to the context.
* @param key the key of field.
* @param value the value of field.
* @return the builder.
*/
public Builder append(String key, String value) {
if (isValid(key) && isValid(value)) {
if (sb.length() > 0) {
sb.append(fieldSeparator);
}
sb.append(key).append(KEY_VALUE_SEPARATOR).append(value);
}
return this;
}
public CallerContext build() {
return new CallerContext(this);
}

10
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -3716,6 +3716,16 @@ The switch to turn S3A auditing on or off.
in audit logs.
</description>
</property>
<property>
<name>hadoop.caller.context.separator</name>
<value>,</value>
<description>
The separator is for context which maybe contain many fields. For example,
if the separator is ',', and there are two key/value fields in context,
in which case the context string is "key1:value1,key2:value2". The
separator should not contain '\t', '\n', '='.
</description>
</property>
<!-- SequenceFile's Sorter properties -->
<property>
<name>seq.io.sort.mb</name>

53
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java Normal file
View File

@ -0,0 +1,53 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ipc;
import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Test;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
public class TestCallerContext {
@Test
public void testBuilderAppend() {
Configuration conf = new Configuration();
conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "$");
CallerContext.Builder builder = new CallerContext.Builder(null, conf);
CallerContext context = builder.append("context1")
.append("context2").append("key3", "value3").build();
Assert.assertEquals(true,
context.getContext().contains("$"));
String[] items = context.getContext().split("\\$");
Assert.assertEquals(3, items.length);
Assert.assertEquals("key3:value3", items[2]);
builder.append("$$");
Assert.assertEquals("context1$context2$key3:value3$$$",
builder.build().getContext());
}
@Test(expected = IllegalArgumentException.class)
public void testNewBuilder() {
Configuration conf = new Configuration();
// Set illegal separator.
conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "\t");
CallerContext.Builder builder = new CallerContext.Builder(null, conf);
builder.build();
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
View File

@ -176,7 +176,7 @@ public void appCreated(RMApp app, long createdTime) {
entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
app.getAppNodeLabelExpression());
if (app.getCallerContext() != null) {
if (app.getCallerContext().getContext() != null) {
if (app.getCallerContext().isContextValid()) {
entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
app.getCallerContext().getContext());
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
View File

@ -127,7 +127,7 @@ public void appCreated(RMApp app, long createdTime) {
ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
app.getAppNodeLabelExpression());
if (app.getCallerContext() != null) {
if (app.getCallerContext().getContext() != null) {
if (app.getCallerContext().isContextValid()) {
entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
app.getCallerContext().getContext());
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
View File

@ -274,16 +274,19 @@ public void setCallerContext(CallerContext callerContext) {
RpcHeaderProtos.RPCCallerContextProto.Builder b = RpcHeaderProtos.RPCCallerContextProto
.newBuilder();
if (callerContext.getContext() != null) {
if (callerContext.isContextValid()) {
b.setContext(callerContext.getContext());
}
if (callerContext.getSignature() != null) {
b.setSignature(ByteString.copyFrom(callerContext.getSignature()));
}
if(callerContext.isContextValid()
|| callerContext.getSignature() != null) {
builder.setCallerContext(b);
}
}
}
@Override
public String toString() {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
View File

@ -177,7 +177,7 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
}
if (callerContext != null) {
if (callerContext.getContext() != null) {
if (callerContext.isContextValid()) {
expLog.append("\tCALLERCONTEXT=context");
}
if (callerContext.getSignature() != null) {
@ -328,7 +328,7 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
}
if (callerContext != null) {
if (callerContext.getContext() != null) {
if (callerContext.isContextValid()) {
expLog.append("\tCALLERCONTEXT=context");
}
if (callerContext.getSignature() != null) {