From 20d8596af2f3ee77beaec766c514c95e41bd2672 Mon Sep 17 00:00:00 2001 From: Alexander Date: Sat, 3 Feb 2024 14:26:30 +0300 Subject: [PATCH] HDFS-17353. Fix failing RBF module tests. (#6491) Contributed by Alexander Bogdanov Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../federation/router/TestRouterRpc.java | 16 +++-- .../router/TestRouterRpcMultiDestination.java | 60 ++++++++++++++++--- 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index 93e905b4ea..766a035151 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -137,6 +137,7 @@ import org.apache.hadoop.test.LambdaTestUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -265,6 +266,12 @@ public static void globalSetUp() throws Exception { .getDatanodeManager().setHeartbeatExpireInterval(3000); } + @After + public void cleanup() { + // clear client context + CallerContext.setCurrent(null); + } + @AfterClass public static void tearDown() { cluster.shutdown(); @@ -2088,10 +2095,10 @@ public void testMkdirsWithCallerContext() throws IOException { // The audit log should contains "callerContext=clientIp:...,clientContext" final String logOutput = auditlog.getOutput(); - assertTrue(logOutput.contains("callerContext=clientIp:")); - assertTrue(logOutput.contains(",clientContext")); - assertTrue(logOutput.contains(",clientId")); - assertTrue(logOutput.contains(",clientCallId")); + assertTrue(logOutput.contains("clientIp:")); + assertTrue(logOutput.contains("clientContext")); + assertTrue(logOutput.contains("clientId")); + assertTrue(logOutput.contains("clientCallId")); assertTrue(verifyFileExists(routerFS, dirPath)); } @@ -2103,7 +2110,6 @@ public void testRealUserPropagationInCallerContext() // Current callerContext is null assertNull(CallerContext.getCurrent()); - UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); UserGroupInformation realUser = UserGroupInformation .createUserForTesting("testRealUser", new String[]{"group"}); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java index 336ea39138..f77ab2e9a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java @@ -33,9 +33,13 @@ import java.io.IOException; import java.lang.reflect.Method; import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; import java.net.URISyntaxException; import java.util.Arrays; import java.util.EnumSet; +import java.util.Enumeration; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Random; @@ -474,21 +478,59 @@ public void testCallerContextWithMultiDestinations() throws IOException { routerFs.getFileStatus(dirPath); String auditFlag = "src=" + dirPath.toString(); - String clientIpInfo = "clientIp:" - + InetAddress.getLocalHost().getHostAddress(); + Set clientIpInfos = getClientIpInfos(); for (String line : auditLog.getOutput().split("\n")) { if (line.contains(auditFlag)) { // assert origin caller context exist in audit log String callerContext = line.substring(line.indexOf("callerContext=")); - assertTrue(callerContext.contains("clientContext")); + assertTrue(String.format("%s doesn't contain 'clientContext'", callerContext), + callerContext.contains("clientContext")); // assert client ip info exist in caller context - assertTrue(callerContext.contains(clientIpInfo)); - // assert client ip info appears only once in caller context - assertEquals(callerContext.indexOf(clientIpInfo), - callerContext.lastIndexOf(clientIpInfo)); + checkCallerContextContainsClientIp(clientIpInfos, callerContext); } } - // clear client context - CallerContext.setCurrent(null); + } + + /** + * Check that one of the IP from all local network interfaces is contained + * only once in callerContext. + * + * @param clientIpInfos IP information extracted from all local network interfaces. + * @param callerContext current caller context. + */ + private static void checkCallerContextContainsClientIp(Set clientIpInfos, + String callerContext) { + String clientIpInfo = null; + for (String curClientIpInfo : clientIpInfos) { + if (callerContext.contains(curClientIpInfo)) { + clientIpInfo = curClientIpInfo; + // assert client ip info appears only once in caller context + assertEquals(String.format("%s contains %s more than once", callerContext, clientIpInfo), + callerContext.indexOf(clientIpInfo), callerContext.lastIndexOf(clientIpInfo)); + break; + } + } + assertNotNull(clientIpInfo); + } + + /** + * A local machine where we run tests may have more than 1 network interface, + * extracting all IP information from them. + * + * @return A set of 'clientIp:IP' where IP is taken from all local network interfaces. + * @throws SocketException + */ + private static Set getClientIpInfos() throws SocketException { + Set clientIpInfos = new HashSet<>(); + Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces(); + while (networkInterfaces.hasMoreElements()) { + NetworkInterface networkInterface = networkInterfaces.nextElement(); + Enumeration inetAddresses = networkInterface.getInetAddresses(); + while (inetAddresses.hasMoreElements()) { + InetAddress inetAddress = inetAddresses.nextElement(); + clientIpInfos.add("clientIp:" + inetAddress.getHostAddress()); + } + } + return clientIpInfos; } } \ No newline at end of file