From f6770dee476900c73ee3dc0a14149a363019a7ba Mon Sep 17 00:00:00 2001 From: LiuGuH <444506464@qq.com> Date: Fri, 19 May 2023 07:32:16 +0800 Subject: [PATCH] HDFS-16979. RBF: Add proxyuser port in hdfsauditlog (#5552). Contributed by liuguanghua. Reviewed-by: Inigo Goiri Reviewed-by: Simbarashe Dzinamarira Signed-off-by: Ayush Saxena --- .../java/org/apache/hadoop/ipc/CallerContext.java | 2 +- .../server/federation/router/TestRouterRpc.java | 7 +++++++ .../hadoop/hdfs/server/namenode/FSNamesystem.java | 14 +++++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java index fec2e36f85..4055599c5f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java @@ -50,7 +50,7 @@ public final class CallerContext { public static final String CLIENT_ID_STR = "clientId"; public static final String CLIENT_CALL_ID_STR = "clientCallId"; public static final String REAL_USER_STR = "realUser"; - + public static final String PROXY_USER_PORT = "proxyUserPort"; /** The caller context. * * It will be truncated if it exceeds the maximum allowed length in diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index d6fe913fee..d3d3421619 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents; @@ -25,6 +27,7 @@ import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists; import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING; +import static org.apache.hadoop.ipc.CallerContext.PROXY_USER_PORT; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertArrayEquals; @@ -209,10 +212,12 @@ public static void globalSetUp() throws Exception { Configuration namenodeConf = new Configuration(); namenodeConf.setBoolean(DFSConfigKeys.HADOOP_CALLER_CONTEXT_ENABLED_KEY, true); + namenodeConf.set(HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY, "256"); // It's very easy to become overloaded for some specific dn in this small // cluster, which will cause the EC file block allocation failure. To avoid // this issue, we disable considerLoad option. namenodeConf.setBoolean(DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); + namenodeConf.setBoolean(DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY, true); cluster = new MiniRouterDFSCluster(false, NUM_SUBCLUSTERS); cluster.setNumDatanodesPerNameservice(NUM_DNS); cluster.addNamenodeOverrides(namenodeConf); @@ -2116,6 +2121,8 @@ public void testRealUserPropagationInCallerContext() // Real user is added to the caller context. assertTrue("The audit log should contain the real user.", logOutput.contains(String.format("realUser:%s", realUser.getUserName()))); + assertTrue("The audit log should contain the proxyuser port.", + logOutput.contains(PROXY_USER_PORT)); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 107439defe..597179a9fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -452,7 +452,7 @@ private void logAuditEvent(boolean succeeded, } private void appendClientPortToCallerContextIfAbsent() { - final CallerContext ctx = CallerContext.getCurrent(); + CallerContext ctx = CallerContext.getCurrent(); if (isClientPortInfoAbsent(ctx)) { String origContext = ctx == null ? null : ctx.getContext(); byte[] origSignature = ctx == null ? null : ctx.getSignature(); @@ -462,6 +462,14 @@ private void appendClientPortToCallerContextIfAbsent() { .setSignature(origSignature) .build()); } + ctx = CallerContext.getCurrent(); + if (isFromProxyUser(ctx)) { + CallerContext.setCurrent( + new CallerContext.Builder(ctx.getContext(), contextFieldSeparator) + .append(CallerContext.PROXY_USER_PORT, String.valueOf(Server.getRemotePort())) + .setSignature(ctx.getSignature()) + .build()); + } } private boolean isClientPortInfoAbsent(CallerContext ctx){ @@ -469,6 +477,10 @@ private boolean isClientPortInfoAbsent(CallerContext ctx){ || !ctx.getContext().contains(CallerContext.CLIENT_PORT_STR); } + private boolean isFromProxyUser(CallerContext ctx) { + return ctx != null && ctx.getContext() != null && + ctx.getContext().contains(CallerContext.REAL_USER_STR); + } /** * Logger for audit events, noting successful FSNamesystem operations. Emits * to FSNamesystem.audit at INFO. Each event causes a set of tab-separated