big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14422. RBF: Router shouldn't allow READ operations in safe mode. Contributed by Inigo Goiri.

Browse Source
This commit is contained in:
Ayush Saxena 2019-04-16 19:45:51 +05:30 committed by Brahma Reddy Battula
parent 0f9b8d7a75
commit de7da9b69e
3 changed files with 74 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
View File

@ -87,6 +87,8 @@ public class MountTableResolver
/** If the tree has been initialized. */
private boolean init = false;
/** If the mount table is manually disabled*/
private boolean disabled = false;
/** Path -> Remote HDFS location. */
private final TreeMap<String, MountTable> tree = new TreeMap<>();
/** Path -> Remote location. */
@ -391,7 +393,14 @@ public PathLocation call() throws Exception {
};
return this.locationCache.get(path, meh);
} catch (ExecutionException e) {
throw new IOException(e);
Throwable cause = e.getCause();
final IOException ioe;
if (cause instanceof IOException) {
ioe = (IOException) cause;
} else {
ioe = new IOException(cause);
}
throw ioe;
} finally {
readLock.unlock();
}
@ -504,7 +513,7 @@ public List<MountTable> getMounts(final String path) throws IOException {
* @throws StateStoreUnavailableException If it cannot connect to the store.
*/
private void verifyMountTable() throws StateStoreUnavailableException {
if (!this.init) {
if (!this.init || disabled) {
throw new StateStoreUnavailableException("Mount Table not initialized");
}
}
@ -654,4 +663,9 @@ public boolean isDefaultNSEnable() {
public void setDefaultNSEnable(boolean defaultNSRWEnable) {
this.defaultNSEnable = defaultNSRWEnable;
}
@VisibleForTesting
public void setDisabled(boolean disable) {
this.disabled = disable;
}
}

15
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
View File

@ -114,6 +114,7 @@
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.hdfs.server.federation.router.security.RouterSecurityManager;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
@ -480,17 +481,26 @@ void checkOperation(OperationCategory op)
// Store the category of the operation category for this thread
opCategory.set(op);
// We allow unchecked and read operations
// We allow unchecked and read operations to try, fail later
if (op == OperationCategory.UNCHECKED || op == OperationCategory.READ) {
return;
}
checkSafeMode();
}
/**
* Check if the Router is in safe mode.
* @throws StandbyException If the Router is in safe mode and cannot serve
* client requests.
*/
private void checkSafeMode() throws StandbyException {
RouterSafemodeService safemodeService = router.getSafemodeService();
if (safemodeService != null && safemodeService.isInSafeMode()) {
// Throw standby exception, router is not available
if (rpcMonitor != null) {
rpcMonitor.routerFailureSafemode();
}
OperationCategory op = opCategory.get();
throw new StandbyException("Router " + router.getRouterId() +
" is in safe mode and cannot handle " + op + " requests");
}
@ -1469,6 +1479,9 @@ protected List<RemoteLocation> getLocationsForPath(String path,
if (this.rpcMonitor != null) {
this.rpcMonitor.routerFailureStateStore();
}
if (ioe instanceof StateStoreUnavailableException) {
checkSafeMode();
}
throw ioe;
}
}

44
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
View File

@ -34,9 +34,13 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
import org.apache.hadoop.hdfs.tools.federation.RouterAdmin;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
@ -234,4 +238,44 @@ private void verifyRouter(RouterServiceState status)
throws IllegalStateException, IOException {
assertEquals(status, router.getRouterState());
}
@Test
public void testRouterNotInitMountTable() throws Exception {
// Manually disable the mount table to trigger unavailable exceptions
MountTableResolver mountTable =
(MountTableResolver)router.getSubclusterResolver();
mountTable.setDisabled(true);
// Wait until it gets out of safe mode
int interval = 2 * (int)conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS);
GenericTestUtils.waitFor(
() -> router.getRouterState() == RouterServiceState.RUNNING,
100, interval);
// Getting file info should fail
try {
router.getRpcServer().getFileInfo("/mnt/file.txt");
fail("We should have thrown StateStoreUnavailableException");
} catch (StateStoreUnavailableException e) {
assertEquals("Mount Table not initialized", e.getMessage());
}
// Enter safe mode
RouterAdminServer admin = router.getAdminServer();
EnterSafeModeRequest request = EnterSafeModeRequest.newInstance();
admin.enterSafeMode(request);
verifyRouter(RouterServiceState.SAFEMODE);
// This time it should report safe mode
try {
router.getRpcServer().getFileInfo("/mnt/file.txt");
fail("We should have thrown a safe mode exception");
} catch (StandbyException e) {
String msg = e.getMessage();
assertTrue("Wrong message: " + msg,
msg.endsWith("is in safe mode and cannot handle READ requests"));
}
}
}