big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17435. Fix TestRouterRpc failed (#6650)

Browse Source 
Reviewed-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
Takanobu Asanuma 2024-03-26 09:38:35 +09:00 committed by GitHub
parent 76489e579b
commit c4f7a3625b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 49 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDFSRouter.java
View File

@ -21,8 +21,16 @@
import org.junit.Test; import org.junit.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.hdfs.server.federation.MockResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.tools.fedbalance.FedBalanceConfigs; import org.apache.hadoop.tools.fedbalance.FedBalanceConfigs;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS;
import static org.junit.Assert.assertEquals;
public class TestDFSRouter { public class TestDFSRouter {
@Test @Test
@ -36,4 +44,45 @@ public void testDefaultConfigs() {
Assert.assertEquals(10, workerThreads); Assert.assertEquals(10, workerThreads);
} }
@Test
public void testClearStaleNamespacesInRouterStateIdContext() throws Exception {
Router testRouter = new Router();
Configuration routerConfig = DFSRouter.getConfiguration();
routerConfig.set(FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS, "2000");
routerConfig.set(RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE, "false");
// Mock resolver classes
routerConfig.setClass(RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
MockResolver.class, ActiveNamenodeResolver.class);
routerConfig.setClass(RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
MockResolver.class, FileSubclusterResolver.class);
testRouter.init(routerConfig);
String nsID1 = "ns0";
String nsID2 = "ns1";
MockResolver resolver = (MockResolver)testRouter.getNamenodeResolver();
resolver.registerNamenode(createNamenodeReport(nsID1, "nn1",
HAServiceProtocol.HAServiceState.ACTIVE));
resolver.registerNamenode(createNamenodeReport(nsID2, "nn1",
HAServiceProtocol.HAServiceState.ACTIVE));
RouterRpcServer rpcServer = testRouter.getRpcServer();
rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID1);
rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID2);
resolver.disableNamespace(nsID1);
Thread.sleep(3000);
RouterStateIdContext context = rpcServer.getRouterStateIdContext();
assertEquals(2, context.getNamespaceIdMap().size());
testRouter.start();
Thread.sleep(3000);
// wait clear stale namespaces
RouterStateIdContext routerStateIdContext = rpcServer.getRouterStateIdContext();
int size = routerStateIdContext.getNamespaceIdMap().size();
assertEquals(1, size);
rpcServer.stop();
rpcServer.close();
testRouter.close();
}
} }

46
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
View File

@ -23,12 +23,10 @@
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING; import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS;
import static org.apache.hadoop.ipc.CallerContext.PROXY_USER_PORT; import static org.apache.hadoop.ipc.CallerContext.PROXY_USER_PORT;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@ -74,7 +72,6 @@
import org.apache.hadoop.fs.SafeModeAction; import org.apache.hadoop.fs.SafeModeAction;
import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DFSTestUtil;
@ -118,7 +115,6 @@
import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics; import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics; import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
import org.apache.hadoop.hdfs.server.federation.metrics.RBFMetrics; import org.apache.hadoop.hdfs.server.federation.metrics.RBFMetrics;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@ -2331,46 +2327,4 @@ public void testGetListingOrder() throws Exception {
fileSystem1.delete(new Path(testPath2), true); fileSystem1.delete(new Path(testPath2), true);
} }
} }
@Test
public void testClearStaleNamespacesInRouterStateIdContext() throws Exception {
Router testRouter = new Router();
Configuration routerConfig = DFSRouter.getConfiguration();
routerConfig.set(FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS, "2000");
routerConfig.set(RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE, "false");
// Mock resolver classes
routerConfig.setClass(RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
MockResolver.class, ActiveNamenodeResolver.class);
routerConfig.setClass(RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
MockResolver.class, FileSubclusterResolver.class);
testRouter.init(routerConfig);
String nsID1 = cluster.getNameservices().get(0);
String nsID2 = cluster.getNameservices().get(1);
MockResolver resolver = (MockResolver)testRouter.getNamenodeResolver();
resolver.registerNamenode(createNamenodeReport(nsID1, "nn1",
HAServiceProtocol.HAServiceState.ACTIVE));
resolver.registerNamenode(createNamenodeReport(nsID2, "nn1",
HAServiceProtocol.HAServiceState.ACTIVE));
RouterRpcServer rpcServer = testRouter.getRpcServer();
rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID1);
rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID2);
resolver.disableNamespace(nsID1);
Thread.sleep(3000);
RouterStateIdContext context = rpcServer.getRouterStateIdContext();
assertEquals(2, context.getNamespaceIdMap().size());
testRouter.start();
Thread.sleep(3000);
// wait clear stale namespaces
RouterStateIdContext routerStateIdContext = rpcServer.getRouterStateIdContext();
int size = routerStateIdContext.getNamespaceIdMap().size();
assertEquals(1, size);
rpcServer.stop();
rpcServer.close();
testRouter.close();
}
} }