big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16495: RBF should prepend the client ip rather than append it.

Browse Source 
Fixes #4054

Signed-off-by: Owen O'Malley <oomalley@linkedin.com>
This commit is contained in:
Owen O'Malley 2022-03-04 16:17:46 -08:00
parent a32cfc2169
commit 7b5eac27ff
No known key found for this signature in database
GPG Key ID: D19EB09DAD1C5877
4 changed files with 32 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
View File

@ -116,7 +116,7 @@ public String toString() {
/** The caller context builder. */ /** The caller context builder. */
public static final class Builder { public static final class Builder {
private static final String KEY_VALUE_SEPARATOR = ":"; public static final String KEY_VALUE_SEPARATOR = ":";
/** /**
* The illegal separators include '\t', '\n', '='. * The illegal separators include '\t', '\n', '='.
* User should not set illegal separator. * User should not set illegal separator.

31
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
View File

@ -466,7 +466,7 @@ private Object invokeMethod(
+ router.getRouterId()); + router.getRouterId());
} }
appendClientIpPortToCallerContextIfAbsent(); addClientIpToCallerContext();
Object ret = null; Object ret = null;
if (rpcMonitor != null) { if (rpcMonitor != null) {
@ -588,19 +588,32 @@ private Object invokeMethod(
/** /**
* For tracking which is the actual client address. * For tracking which is the actual client address.
* It adds trace info "clientIp:ip" and "clientPort:port" * It adds trace info "clientIp:ip" and "clientPort:port"
* to caller context if they are absent. * in the caller context, removing the old values if they were
* already present.
*/ */
private void appendClientIpPortToCallerContextIfAbsent() { private void addClientIpToCallerContext() {
CallerContext ctx = CallerContext.getCurrent(); CallerContext ctx = CallerContext.getCurrent();
String origContext = ctx == null ? null : ctx.getContext(); String origContext = ctx == null ? null : ctx.getContext();
byte[] origSignature = ctx == null ? null : ctx.getSignature(); byte[] origSignature = ctx == null ? null : ctx.getSignature();
CallerContext.setCurrent( CallerContext.Builder builder =
new CallerContext.Builder(origContext, contextFieldSeparator) new CallerContext.Builder("", contextFieldSeparator)
.appendIfAbsent(CLIENT_IP_STR, Server.getRemoteAddress()) .append(CLIENT_IP_STR, Server.getRemoteAddress())
.appendIfAbsent(CLIENT_PORT_STR, .append(CLIENT_PORT_STR,
Integer.toString(Server.getRemotePort())) Integer.toString(Server.getRemotePort()))
.setSignature(origSignature) .setSignature(origSignature);
.build()); // Append the original caller context
if (origContext != null) {
for (String part : origContext.split(contextFieldSeparator)) {
String[] keyValue =
part.split(CallerContext.Builder.KEY_VALUE_SEPARATOR, 2);
if (keyValue.length == 2) {
builder.appendIfAbsent(keyValue[0], keyValue[1]);
} else if (keyValue.length == 1) {
builder.append(keyValue[0]);
}
}
}
CallerContext.setCurrent(builder.build());
} }
/** /**

13
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
View File

@ -1950,9 +1950,10 @@ public void testMkdirsWithCallerContext() throws IOException {
FsPermission permission = new FsPermission("755"); FsPermission permission = new FsPermission("755");
routerProtocol.mkdirs(dirPath, permission, false); routerProtocol.mkdirs(dirPath, permission, false);
// The audit log should contains "callerContext=clientContext,clientIp:" // The audit log should contains "callerContext=clientIp:...,clientContext"
assertTrue(auditlog.getOutput() final String logOutput = auditlog.getOutput();
.contains("callerContext=clientContext,clientIp:")); assertTrue(logOutput.contains("callerContext=clientIp:"));
assertTrue(logOutput.contains(",clientContext"));
assertTrue(verifyFileExists(routerFS, dirPath)); assertTrue(verifyFileExists(routerFS, dirPath));
} }
@ -1997,9 +1998,9 @@ public void testAddClientIpPortToCallerContext() throws IOException {
// Create a directory via the router. // Create a directory via the router.
routerProtocol.getFileInfo(dirPath); routerProtocol.getFileInfo(dirPath);
// The audit log should contains the original clientIp and clientPort // The audit log should not contain the original clientIp and clientPort
// set by client. // set by client.
assertTrue(auditLog.getOutput().contains("clientIp:1.1.1.1")); assertFalse(auditLog.getOutput().contains("clientIp:1.1.1.1"));
assertTrue(auditLog.getOutput().contains("clientPort:1234")); assertFalse(auditLog.getOutput().contains("clientPort:1234"));
} }
} }

5
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
View File

@ -464,9 +464,8 @@ public void testCallerContextWithMultiDestinations() throws IOException {
for (String line : auditLog.getOutput().split("\n")) { for (String line : auditLog.getOutput().split("\n")) {
if (line.contains(auditFlag)) { if (line.contains(auditFlag)) {
// assert origin caller context exist in audit log // assert origin caller context exist in audit log
assertTrue(line.contains("callerContext=clientContext")); String callerContext = line.substring(line.indexOf("callerContext="));
String callerContext = line.substring( assertTrue(callerContext.contains("clientContext"));
line.indexOf("callerContext=clientContext"));
// assert client ip info exist in caller context // assert client ip info exist in caller context
assertTrue(callerContext.contains(clientIpInfo)); assertTrue(callerContext.contains(clientIpInfo));
// assert client ip info appears only once in caller context // assert client ip info appears only once in caller context