big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17353. Fix failing RBF module tests. (#6491) Contributed by Alexander Bogdanov

Browse Source 
Reviewed-by: Inigo Goiri <inigoiri@apache.org>
Signed-off-by: Shilun Fan <slfan1989@apache.org>
This commit is contained in:
Alexander 2024-02-03 14:26:30 +03:00 committed by GitHub
parent 6464507cd1
commit 20d8596af2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 62 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
View File

@ -137,6 +137,7 @@
import org.apache.hadoop.test.LambdaTestUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@ -265,6 +266,12 @@ public static void globalSetUp() throws Exception {
.getDatanodeManager().setHeartbeatExpireInterval(3000);
}
@After
public void cleanup() {
// clear client context
CallerContext.setCurrent(null);
}
@AfterClass
public static void tearDown() {
cluster.shutdown();
@ -2088,10 +2095,10 @@ public void testMkdirsWithCallerContext() throws IOException {
// The audit log should contains "callerContext=clientIp:...,clientContext"
final String logOutput = auditlog.getOutput();
assertTrue(logOutput.contains("callerContext=clientIp:"));
assertTrue(logOutput.contains(",clientContext"));
assertTrue(logOutput.contains(",clientId"));
assertTrue(logOutput.contains(",clientCallId"));
assertTrue(logOutput.contains("clientIp:"));
assertTrue(logOutput.contains("clientContext"));
assertTrue(logOutput.contains("clientId"));
assertTrue(logOutput.contains("clientCallId"));
assertTrue(verifyFileExists(routerFS, dirPath));
}
@ -2103,7 +2110,6 @@ public void testRealUserPropagationInCallerContext()
// Current callerContext is null
assertNull(CallerContext.getCurrent());
UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
UserGroupInformation realUser = UserGroupInformation
.createUserForTesting("testRealUser", new String[]{"group"});

58
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
View File

@ -33,9 +33,13 @@
import java.io.IOException;
import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
@ -474,21 +478,59 @@ public void testCallerContextWithMultiDestinations() throws IOException {
routerFs.getFileStatus(dirPath);
String auditFlag = "src=" + dirPath.toString();
String clientIpInfo = "clientIp:"
+ InetAddress.getLocalHost().getHostAddress();
Set<String> clientIpInfos = getClientIpInfos();
for (String line : auditLog.getOutput().split("\n")) {
if (line.contains(auditFlag)) {
// assert origin caller context exist in audit log
String callerContext = line.substring(line.indexOf("callerContext="));
assertTrue(callerContext.contains("clientContext"));
assertTrue(String.format("%s doesn't contain 'clientContext'", callerContext),
callerContext.contains("clientContext"));
// assert client ip info exist in caller context
assertTrue(callerContext.contains(clientIpInfo));
checkCallerContextContainsClientIp(clientIpInfos, callerContext);
}
}
}
/**
* Check that one of the IP from all local network interfaces is contained
* only once in callerContext.
*
* @param clientIpInfos IP information extracted from all local network interfaces.
* @param callerContext current caller context.
*/
private static void checkCallerContextContainsClientIp(Set<String> clientIpInfos,
String callerContext) {
String clientIpInfo = null;
for (String curClientIpInfo : clientIpInfos) {
if (callerContext.contains(curClientIpInfo)) {
clientIpInfo = curClientIpInfo;
// assert client ip info appears only once in caller context
assertEquals(callerContext.indexOf(clientIpInfo),
callerContext.lastIndexOf(clientIpInfo));
assertEquals(String.format("%s contains %s more than once", callerContext, clientIpInfo),
callerContext.indexOf(clientIpInfo), callerContext.lastIndexOf(clientIpInfo));
break;
}
}
// clear client context
CallerContext.setCurrent(null);
assertNotNull(clientIpInfo);
}
/**
* A local machine where we run tests may have more than 1 network interface,
* extracting all IP information from them.
*
* @return A set of 'clientIp:IP' where IP is taken from all local network interfaces.
* @throws SocketException
*/
private static Set<String> getClientIpInfos() throws SocketException {
Set<String> clientIpInfos = new HashSet<>();
Enumeration<NetworkInterface> networkInterfaces = NetworkInterface.getNetworkInterfaces();
while (networkInterfaces.hasMoreElements()) {
NetworkInterface networkInterface = networkInterfaces.nextElement();
Enumeration<InetAddress> inetAddresses = networkInterface.getInetAddresses();
while (inetAddresses.hasMoreElements()) {
InetAddress inetAddress = inetAddresses.nextElement();
clientIpInfos.add("clientIp:" + inetAddress.getHostAddress());
}
}
return clientIpInfos;
}
}