HDFS-14422. RBF: Router shouldn't allow READ operations in safe mode. Contributed by Inigo Goiri.

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java

@@ -87,6 +87,8 @@ public class MountTableResolver
 
   /** If the tree has been initialized. */
   private boolean init = false;
+  /** If the mount table is manually disabled*/
+  private boolean disabled = false;
   /** Path -> Remote HDFS location. */
   private final TreeMap<String, MountTable> tree = new TreeMap<>();
   /** Path -> Remote location. */
@@ -391,7 +393,14 @@ public PathLocation call() throws Exception {
       };
       return this.locationCache.get(path, meh);
     } catch (ExecutionException e) {
-      throw new IOException(e);
+      Throwable cause = e.getCause();
+      final IOException ioe;
+      if (cause instanceof IOException) {
+        ioe = (IOException) cause;
+      } else {
+        ioe = new IOException(cause);
+      }
+      throw ioe;
     } finally {
       readLock.unlock();
     }
@@ -504,7 +513,7 @@ public List<MountTable> getMounts(final String path) throws IOException {
    * @throws StateStoreUnavailableException If it cannot connect to the store.
    */
   private void verifyMountTable() throws StateStoreUnavailableException {
-    if (!this.init) {
+    if (!this.init || disabled) {
       throw new StateStoreUnavailableException("Mount Table not initialized");
     }
   }
@@ -654,4 +663,9 @@ public boolean isDefaultNSEnable() {
   public void setDefaultNSEnable(boolean defaultNSRWEnable) {
     this.defaultNSEnable = defaultNSRWEnable;
   }
+
+  @VisibleForTesting
+  public void setDisabled(boolean disable) {
+    this.disabled = disable;
+  }
 }

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java

@@ -114,6 +114,7 @@
 import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
 import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
 import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
 import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
 import org.apache.hadoop.hdfs.server.federation.router.security.RouterSecurityManager;
 import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
@@ -480,17 +481,26 @@ void checkOperation(OperationCategory op)
     // Store the category of the operation category for this thread
     opCategory.set(op);
 
-    // We allow unchecked and read operations
+    // We allow unchecked and read operations to try, fail later
     if (op == OperationCategory.UNCHECKED || op == OperationCategory.READ) {
       return;
     }
+    checkSafeMode();
+  }
 
+  /**
+   * Check if the Router is in safe mode.
+   * @throws StandbyException If the Router is in safe mode and cannot serve
+   *                          client requests.
+   */
+  private void checkSafeMode() throws StandbyException {
     RouterSafemodeService safemodeService = router.getSafemodeService();
     if (safemodeService != null && safemodeService.isInSafeMode()) {
       // Throw standby exception, router is not available
       if (rpcMonitor != null) {
         rpcMonitor.routerFailureSafemode();
       }
+      OperationCategory op = opCategory.get();
       throw new StandbyException("Router " + router.getRouterId() +
           " is in safe mode and cannot handle " + op + " requests");
     }
@@ -1469,6 +1479,9 @@ protected List<RemoteLocation> getLocationsForPath(String path,
       if (this.rpcMonitor != null) {
         this.rpcMonitor.routerFailureStateStore();
       }
+      if (ioe instanceof StateStoreUnavailableException) {
+        checkSafeMode();
+      }
       throw ioe;
     }
   }

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java

@@ -34,9 +34,13 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
 import org.apache.hadoop.hdfs.tools.federation.RouterAdmin;
 import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.ToolRunner;
 import org.junit.After;
@@ -234,4 +238,44 @@ private void verifyRouter(RouterServiceState status)
       throws IllegalStateException, IOException {
     assertEquals(status, router.getRouterState());
   }
+
+  @Test
+  public void testRouterNotInitMountTable() throws Exception {
+
+    // Manually disable the mount table to trigger unavailable exceptions
+    MountTableResolver mountTable =
+        (MountTableResolver)router.getSubclusterResolver();
+    mountTable.setDisabled(true);
+
+    // Wait until it gets out of safe mode
+    int interval = 2 * (int)conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
+        TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS);
+    GenericTestUtils.waitFor(
+        () -> router.getRouterState() == RouterServiceState.RUNNING,
+        100, interval);
+
+    // Getting file info should fail
+    try {
+      router.getRpcServer().getFileInfo("/mnt/file.txt");
+      fail("We should have thrown StateStoreUnavailableException");
+    } catch (StateStoreUnavailableException e) {
+      assertEquals("Mount Table not initialized", e.getMessage());
+    }
+
+    // Enter safe mode
+    RouterAdminServer admin = router.getAdminServer();
+    EnterSafeModeRequest request = EnterSafeModeRequest.newInstance();
+    admin.enterSafeMode(request);
+    verifyRouter(RouterServiceState.SAFEMODE);
+
+    // This time it should report safe mode
+    try {
+      router.getRpcServer().getFileInfo("/mnt/file.txt");
+      fail("We should have thrown a safe mode exception");
+    } catch (StandbyException e) {
+      String msg = e.getMessage();
+      assertTrue("Wrong message: " + msg,
+          msg.endsWith("is in safe mode and cannot handle READ requests"));
+    }
+  }
 }