HADOOP-17276. Extend CallerContext to make it include many items (#2327)

Cherry-picked from d0d10f7e by Owen O'Malley

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java

@@ -371,6 +371,9 @@ public class CommonConfigurationKeysPublic {
       "hadoop.caller.context.signature.max.size";
   public static final int     HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT =
       40;
+  public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_KEY =
+      "hadoop.caller.context.separator";
+  public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT = ",";
 
   /**
    * @see

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java

@@ -21,10 +21,17 @@
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
 
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
 
 /**
  * A class defining the caller context for auditing coarse granularity
@@ -54,8 +61,8 @@ public final class CallerContext {
   private final byte[] signature;
 
   private CallerContext(Builder builder) {
-    this.context = builder.context;
+    this.context = builder.getContext();
-    this.signature = builder.signature;
+    this.signature = builder.getSignature();
   }
 
   public String getContext() {
@@ -109,11 +116,53 @@ public String toString() {
 
   /** The caller context builder. */
   public static final class Builder {
-    private final String context;
+    private static final String KEY_VALUE_SEPARATOR = ":";
+    /**
+     * The illegal separators include '\t', '\n', '='.
+     * User should not set illegal separator.
+     */
+    private static final Set<String> ILLEGAL_SEPARATORS =
+        Collections.unmodifiableSet(
+            new HashSet<>(Arrays.asList("\t", "\n", "=")));
+    private final String fieldSeparator;
+    private final StringBuilder sb = new StringBuilder();
     private byte[] signature;
 
     public Builder(String context) {
-      this.context = context;
+      this(context, new Configuration());
+    }
+
+    public Builder(String context, Configuration conf) {
+      if (isValid(context)) {
+        sb.append(context);
+      }
+      fieldSeparator = conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
+          HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
+      checkFieldSeparator(fieldSeparator);
+    }
+
+    /**
+     * Check whether the separator is legal.
+     * The illegal separators include '\t', '\n', '='.
+     * Throw IllegalArgumentException if the separator is Illegal.
+     * @param separator the separator of fields.
+     */
+    private void checkFieldSeparator(String separator) {
+      if (ILLEGAL_SEPARATORS.contains(separator)) {
+        throw new IllegalArgumentException("Illegal field separator: "
+            + separator);
+      }
+    }
+
+    /**
+     * Whether the field is valid.
+     * The field should not contain '\t', '\n', '='.
+     * Because the context could be written to audit log.
+     * @param field one of the fields in context.
+     * @return true if the field is not null or empty.
+     */
+    private boolean isValid(String field) {
+      return field != null && field.length() > 0;
     }
 
     public Builder setSignature(byte[] signature) {
@@ -123,6 +172,54 @@ public Builder setSignature(byte[] signature) {
       return this;
     }
 
+    /**
+     * Get the context.
+     * For example, the context is "key1:value1,key2:value2".
+     * @return the valid context or null.
+     */
+    public String getContext() {
+      return sb.length() > 0 ? sb.toString() : null;
+    }
+
+    /**
+     * Get the signature.
+     * @return the signature.
+     */
+    public byte[] getSignature() {
+      return signature;
+    }
+
+    /**
+     * Append new field to the context.
+     * @param field one of fields to append.
+     * @return the builder.
+     */
+    public Builder append(String field) {
+      if (isValid(field)) {
+        if (sb.length() > 0) {
+          sb.append(fieldSeparator);
+        }
+        sb.append(field);
+      }
+      return this;
+    }
+
+    /**
+     * Append new field which contains key and value to the context.
+     * @param key the key of field.
+     * @param value the value of field.
+     * @return the builder.
+     */
+    public Builder append(String key, String value) {
+      if (isValid(key) && isValid(value)) {
+        if (sb.length() > 0) {
+          sb.append(fieldSeparator);
+        }
+        sb.append(key).append(KEY_VALUE_SEPARATOR).append(value);
+      }
+      return this;
+    }
+
     public CallerContext build() {
       return new CallerContext(this);
     }

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -3716,6 +3716,16 @@ The switch to turn S3A auditing on or off.
       in audit logs.
     </description>
   </property>
+  <property>
+    <name>hadoop.caller.context.separator</name>
+    <value>,</value>
+    <description>
+      The separator is for context which maybe contain many fields. For example,
+      if the separator is ',',  and there are two key/value fields in context,
+      in which case the context string is "key1:value1,key2:value2". The
+      separator should not contain '\t', '\n', '='.
+    </description>
+  </property>
 <!-- SequenceFile's Sorter properties -->
   <property>
     <name>seq.io.sort.mb</name>

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
+
+public class TestCallerContext {
+  @Test
+  public void testBuilderAppend() {
+    Configuration conf = new Configuration();
+    conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "$");
+    CallerContext.Builder builder = new CallerContext.Builder(null, conf);
+    CallerContext context = builder.append("context1")
+        .append("context2").append("key3", "value3").build();
+    Assert.assertEquals(true,
+        context.getContext().contains("$"));
+    String[] items = context.getContext().split("\\$");
+    Assert.assertEquals(3, items.length);
+    Assert.assertEquals("key3:value3", items[2]);
+
+    builder.append("$$");
+    Assert.assertEquals("context1$context2$key3:value3$$$",
+        builder.build().getContext());
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testNewBuilder() {
+    Configuration conf = new Configuration();
+    // Set illegal separator.
+    conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "\t");
+    CallerContext.Builder builder = new CallerContext.Builder(null, conf);
+    builder.build();
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java

@@ -176,7 +176,7 @@ public void appCreated(RMApp app, long createdTime) {
     entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
         app.getAppNodeLabelExpression());
     if (app.getCallerContext() != null) {
-      if (app.getCallerContext().getContext() != null) {
+      if (app.getCallerContext().isContextValid()) {
         entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
             app.getCallerContext().getContext());
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java

@@ -127,7 +127,7 @@ public void appCreated(RMApp app, long createdTime) {
         ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
         app.getAppNodeLabelExpression());
     if (app.getCallerContext() != null) {
-      if (app.getCallerContext().getContext() != null) {
+      if (app.getCallerContext().isContextValid()) {
         entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
             app.getCallerContext().getContext());
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java

@@ -274,16 +274,19 @@ public void setCallerContext(CallerContext callerContext) {
 
       RpcHeaderProtos.RPCCallerContextProto.Builder b = RpcHeaderProtos.RPCCallerContextProto
           .newBuilder();
-      if (callerContext.getContext() != null) {
+      if (callerContext.isContextValid()) {
         b.setContext(callerContext.getContext());
       }
       if (callerContext.getSignature() != null) {
         b.setSignature(ByteString.copyFrom(callerContext.getSignature()));
       }
 
+      if(callerContext.isContextValid()
+          || callerContext.getSignature() != null) {
         builder.setCallerContext(b);
       }
     }
+  }
 
   @Override
   public String toString() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java

@@ -177,7 +177,7 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
       expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
     }
     if (callerContext != null) {
-      if (callerContext.getContext() != null) {
+      if (callerContext.isContextValid()) {
         expLog.append("\tCALLERCONTEXT=context");
       }
       if (callerContext.getSignature() != null) {
@@ -328,7 +328,7 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
       expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
     }
     if (callerContext != null) {
-      if (callerContext.getContext() != null) {
+      if (callerContext.isContextValid()) {
         expLog.append("\tCALLERCONTEXT=context");
       }
       if (callerContext.getSignature() != null) {