diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java index 03b051db34..8baa5e22a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java @@ -87,6 +87,8 @@ public class MountTableResolver /** If the tree has been initialized. */ private boolean init = false; + /** If the mount table is manually disabled*/ + private boolean disabled = false; /** Path -> Remote HDFS location. */ private final TreeMap tree = new TreeMap<>(); /** Path -> Remote location. */ @@ -391,7 +393,14 @@ public PathLocation call() throws Exception { }; return this.locationCache.get(path, meh); } catch (ExecutionException e) { - throw new IOException(e); + Throwable cause = e.getCause(); + final IOException ioe; + if (cause instanceof IOException) { + ioe = (IOException) cause; + } else { + ioe = new IOException(cause); + } + throw ioe; } finally { readLock.unlock(); } @@ -504,7 +513,7 @@ public List getMounts(final String path) throws IOException { * @throws StateStoreUnavailableException If it cannot connect to the store. */ private void verifyMountTable() throws StateStoreUnavailableException { - if (!this.init) { + if (!this.init || disabled) { throw new StateStoreUnavailableException("Mount Table not initialized"); } } @@ -654,4 +663,9 @@ public boolean isDefaultNSEnable() { public void setDefaultNSEnable(boolean defaultNSRWEnable) { this.defaultNSEnable = defaultNSRWEnable; } + + @VisibleForTesting + public void setDisabled(boolean disable) { + this.disabled = disable; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index b934355dc9..3a2f910da4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -114,6 +114,7 @@ import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; +import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; import org.apache.hadoop.hdfs.server.federation.router.security.RouterSecurityManager; import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature; @@ -480,17 +481,26 @@ void checkOperation(OperationCategory op) // Store the category of the operation category for this thread opCategory.set(op); - // We allow unchecked and read operations + // We allow unchecked and read operations to try, fail later if (op == OperationCategory.UNCHECKED || op == OperationCategory.READ) { return; } + checkSafeMode(); + } + /** + * Check if the Router is in safe mode. + * @throws StandbyException If the Router is in safe mode and cannot serve + * client requests. + */ + private void checkSafeMode() throws StandbyException { RouterSafemodeService safemodeService = router.getSafemodeService(); if (safemodeService != null && safemodeService.isInSafeMode()) { // Throw standby exception, router is not available if (rpcMonitor != null) { rpcMonitor.routerFailureSafemode(); } + OperationCategory op = opCategory.get(); throw new StandbyException("Router " + router.getRouterId() + " is in safe mode and cannot handle " + op + " requests"); } @@ -1469,6 +1479,9 @@ protected List getLocationsForPath(String path, if (this.rpcMonitor != null) { this.rpcMonitor.routerFailureStateStore(); } + if (ioe instanceof StateStoreUnavailableException) { + checkSafeMode(); + } throw ioe; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java index 9c1aeb2b3f..75104bd193 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java @@ -34,9 +34,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; +import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException; +import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest; import org.apache.hadoop.hdfs.tools.federation.RouterAdmin; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.service.Service.STATE; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.ToolRunner; import org.junit.After; @@ -234,4 +238,44 @@ private void verifyRouter(RouterServiceState status) throws IllegalStateException, IOException { assertEquals(status, router.getRouterState()); } + + @Test + public void testRouterNotInitMountTable() throws Exception { + + // Manually disable the mount table to trigger unavailable exceptions + MountTableResolver mountTable = + (MountTableResolver)router.getSubclusterResolver(); + mountTable.setDisabled(true); + + // Wait until it gets out of safe mode + int interval = 2 * (int)conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION, + TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS); + GenericTestUtils.waitFor( + () -> router.getRouterState() == RouterServiceState.RUNNING, + 100, interval); + + // Getting file info should fail + try { + router.getRpcServer().getFileInfo("/mnt/file.txt"); + fail("We should have thrown StateStoreUnavailableException"); + } catch (StateStoreUnavailableException e) { + assertEquals("Mount Table not initialized", e.getMessage()); + } + + // Enter safe mode + RouterAdminServer admin = router.getAdminServer(); + EnterSafeModeRequest request = EnterSafeModeRequest.newInstance(); + admin.enterSafeMode(request); + verifyRouter(RouterServiceState.SAFEMODE); + + // This time it should report safe mode + try { + router.getRpcServer().getFileInfo("/mnt/file.txt"); + fail("We should have thrown a safe mode exception"); + } catch (StandbyException e) { + String msg = e.getMessage(); + assertTrue("Wrong message: " + msg, + msg.endsWith("is in safe mode and cannot handle READ requests")); + } + } }