HADOOP-17276. Extend CallerContext to make it include many items (#2327)
This commit is contained in:
parent
18fa4397e6
commit
d0d10f7e8f
@ -352,6 +352,9 @@ public class CommonConfigurationKeysPublic {
|
||||
"hadoop.caller.context.signature.max.size";
|
||||
public static final int HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT =
|
||||
40;
|
||||
public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_KEY =
|
||||
"hadoop.caller.context.separator";
|
||||
public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT = ",";
|
||||
|
||||
/**
|
||||
* @see
|
||||
|
@ -21,10 +21,17 @@
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT;
|
||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
|
||||
|
||||
/**
|
||||
* A class defining the caller context for auditing coarse granularity
|
||||
@ -54,8 +61,8 @@ public final class CallerContext {
|
||||
private final byte[] signature;
|
||||
|
||||
private CallerContext(Builder builder) {
|
||||
this.context = builder.context;
|
||||
this.signature = builder.signature;
|
||||
this.context = builder.getContext();
|
||||
this.signature = builder.getSignature();
|
||||
}
|
||||
|
||||
public String getContext() {
|
||||
@ -109,11 +116,53 @@ public String toString() {
|
||||
|
||||
/** The caller context builder. */
|
||||
public static final class Builder {
|
||||
private final String context;
|
||||
private static final String KEY_VALUE_SEPARATOR = ":";
|
||||
/**
|
||||
* The illegal separators include '\t', '\n', '='.
|
||||
* User should not set illegal separator.
|
||||
*/
|
||||
private static final Set<String> ILLEGAL_SEPARATORS =
|
||||
Collections.unmodifiableSet(
|
||||
new HashSet<>(Arrays.asList("\t", "\n", "=")));
|
||||
private final String fieldSeparator;
|
||||
private final StringBuilder sb = new StringBuilder();
|
||||
private byte[] signature;
|
||||
|
||||
public Builder(String context) {
|
||||
this.context = context;
|
||||
this(context, new Configuration());
|
||||
}
|
||||
|
||||
public Builder(String context, Configuration conf) {
|
||||
if (isValid(context)) {
|
||||
sb.append(context);
|
||||
}
|
||||
fieldSeparator = conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
|
||||
HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
|
||||
checkFieldSeparator(fieldSeparator);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the separator is legal.
|
||||
* The illegal separators include '\t', '\n', '='.
|
||||
* Throw IllegalArgumentException if the separator is Illegal.
|
||||
* @param separator the separator of fields.
|
||||
*/
|
||||
private void checkFieldSeparator(String separator) {
|
||||
if (ILLEGAL_SEPARATORS.contains(separator)) {
|
||||
throw new IllegalArgumentException("Illegal field separator: "
|
||||
+ separator);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the field is valid.
|
||||
* The field should not contain '\t', '\n', '='.
|
||||
* Because the context could be written to audit log.
|
||||
* @param field one of the fields in context.
|
||||
* @return true if the field is not null or empty.
|
||||
*/
|
||||
private boolean isValid(String field) {
|
||||
return field != null && field.length() > 0;
|
||||
}
|
||||
|
||||
public Builder setSignature(byte[] signature) {
|
||||
@ -123,6 +172,54 @@ public Builder setSignature(byte[] signature) {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the context.
|
||||
* For example, the context is "key1:value1,key2:value2".
|
||||
* @return the valid context or null.
|
||||
*/
|
||||
public String getContext() {
|
||||
return sb.length() > 0 ? sb.toString() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the signature.
|
||||
* @return the signature.
|
||||
*/
|
||||
public byte[] getSignature() {
|
||||
return signature;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append new field to the context.
|
||||
* @param field one of fields to append.
|
||||
* @return the builder.
|
||||
*/
|
||||
public Builder append(String field) {
|
||||
if (isValid(field)) {
|
||||
if (sb.length() > 0) {
|
||||
sb.append(fieldSeparator);
|
||||
}
|
||||
sb.append(field);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append new field which contains key and value to the context.
|
||||
* @param key the key of field.
|
||||
* @param value the value of field.
|
||||
* @return the builder.
|
||||
*/
|
||||
public Builder append(String key, String value) {
|
||||
if (isValid(key) && isValid(value)) {
|
||||
if (sb.length() > 0) {
|
||||
sb.append(fieldSeparator);
|
||||
}
|
||||
sb.append(key).append(KEY_VALUE_SEPARATOR).append(value);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public CallerContext build() {
|
||||
return new CallerContext(this);
|
||||
}
|
||||
|
@ -3847,6 +3847,16 @@
|
||||
in audit logs.
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hadoop.caller.context.separator</name>
|
||||
<value>,</value>
|
||||
<description>
|
||||
The separator is for context which maybe contain many fields. For example,
|
||||
if the separator is ',', and there are two key/value fields in context,
|
||||
in which case the context string is "key1:value1,key2:value2". The
|
||||
separator should not contain '\t', '\n', '='.
|
||||
</description>
|
||||
</property>
|
||||
<!-- SequenceFile's Sorter properties -->
|
||||
<property>
|
||||
<name>seq.io.sort.mb</name>
|
||||
|
@ -0,0 +1,53 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.ipc;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
|
||||
|
||||
public class TestCallerContext {
|
||||
@Test
|
||||
public void testBuilderAppend() {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "$");
|
||||
CallerContext.Builder builder = new CallerContext.Builder(null, conf);
|
||||
CallerContext context = builder.append("context1")
|
||||
.append("context2").append("key3", "value3").build();
|
||||
Assert.assertEquals(true,
|
||||
context.getContext().contains("$"));
|
||||
String[] items = context.getContext().split("\\$");
|
||||
Assert.assertEquals(3, items.length);
|
||||
Assert.assertEquals("key3:value3", items[2]);
|
||||
|
||||
builder.append("$$");
|
||||
Assert.assertEquals("context1$context2$key3:value3$$$",
|
||||
builder.build().getContext());
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void testNewBuilder() {
|
||||
Configuration conf = new Configuration();
|
||||
// Set illegal separator.
|
||||
conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "\t");
|
||||
CallerContext.Builder builder = new CallerContext.Builder(null, conf);
|
||||
builder.build();
|
||||
}
|
||||
}
|
@ -94,7 +94,7 @@ public void appCreated(RMApp app, long createdTime) {
|
||||
entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
|
||||
app.getAppNodeLabelExpression());
|
||||
if (app.getCallerContext() != null) {
|
||||
if (app.getCallerContext().getContext() != null) {
|
||||
if (app.getCallerContext().isContextValid()) {
|
||||
entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
|
||||
app.getCallerContext().getContext());
|
||||
}
|
||||
|
@ -127,7 +127,7 @@ public void appCreated(RMApp app, long createdTime) {
|
||||
ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
|
||||
app.getAppNodeLabelExpression());
|
||||
if (app.getCallerContext() != null) {
|
||||
if (app.getCallerContext().getContext() != null) {
|
||||
if (app.getCallerContext().isContextValid()) {
|
||||
entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
|
||||
app.getCallerContext().getContext());
|
||||
}
|
||||
|
@ -259,14 +259,17 @@ public void setCallerContext(CallerContext callerContext) {
|
||||
|
||||
RpcHeaderProtos.RPCCallerContextProto.Builder b = RpcHeaderProtos.RPCCallerContextProto
|
||||
.newBuilder();
|
||||
if (callerContext.getContext() != null) {
|
||||
if (callerContext.isContextValid()) {
|
||||
b.setContext(callerContext.getContext());
|
||||
}
|
||||
if (callerContext.getSignature() != null) {
|
||||
b.setSignature(ByteString.copyFrom(callerContext.getSignature()));
|
||||
}
|
||||
|
||||
builder.setCallerContext(b);
|
||||
if(callerContext.isContextValid()
|
||||
|| callerContext.getSignature() != null) {
|
||||
builder.setCallerContext(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -177,7 +177,7 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
|
||||
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
|
||||
}
|
||||
if (callerContext != null) {
|
||||
if (callerContext.getContext() != null) {
|
||||
if (callerContext.isContextValid()) {
|
||||
expLog.append("\tCALLERCONTEXT=context");
|
||||
}
|
||||
if (callerContext.getSignature() != null) {
|
||||
@ -328,7 +328,7 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
|
||||
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
|
||||
}
|
||||
if (callerContext != null) {
|
||||
if (callerContext.getContext() != null) {
|
||||
if (callerContext.isContextValid()) {
|
||||
expLog.append("\tCALLERCONTEXT=context");
|
||||
}
|
||||
if (callerContext.getSignature() != null) {
|
||||
|
Loading…
Reference in New Issue
Block a user