big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13857. RBF: Choose to enable the default nameservice to read/write files. Contributed by yanghuafeng.

Browse Source
This commit is contained in:
Inigo Goiri 2018-09-04 12:17:17 -07:00
parent 6bbd249011
commit 54f2044595
5 changed files with 76 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
View File

@ -20,6 +20,8 @@
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMESERVICE_ID;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE_DEFAULT;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE;
@ -95,6 +97,8 @@ public class MountTableResolver
/** Default nameservice when no mount matches the math. */
private String defaultNameService = "";
/** If use default nameservice to read and write files. */
private boolean defaultNSEnable = true;
/** Synchronization for both the tree and the cache. */
private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
@ -163,6 +167,10 @@ private void initDefaultNameService(Configuration conf) {
DFS_ROUTER_DEFAULT_NAMESERVICE,
DFSUtil.getNamenodeNameServiceId(conf));
this.defaultNSEnable = conf.getBoolean(
DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE,
DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE_DEFAULT);
if (defaultNameService == null) {
LOG.warn(
"{} and {} is not set. Fallback to {} as the default name service.",
@ -176,9 +184,12 @@ private void initDefaultNameService(Configuration conf) {
}
if (this.defaultNameService.equals("")) {
this.defaultNSEnable = false;
LOG.warn("Default name service is not set.");
} else {
LOG.info("Default name service: {}", this.defaultNameService);
String enable = this.defaultNSEnable ? "enabled" : "disabled";
LOG.info("Default name service: {}, {} to read or write",
this.defaultNameService, enable);
}
}
@ -406,13 +417,17 @@ public PathLocation call() throws Exception {
* @param path Path to check/insert.
* @return New remote location.
*/
public PathLocation lookupLocation(final String path) {
public PathLocation lookupLocation(final String path) throws IOException {
PathLocation ret = null;
MountTable entry = findDeepest(path);
if (entry != null) {
ret = buildLocation(path, entry);
} else {
// Not found, use default location
if (!defaultNSEnable) {
throw new IOException("Cannot find locations for " + path + ", " +
"because the default nameservice is disabled to read or write");
}
RemoteLocation remoteLocation =
new RemoteLocation(defaultNameService, path, path);
List<RemoteLocation> locations =
@ -623,4 +638,24 @@ protected long getCacheSize() throws IOException{
}
throw new IOException("localCache is null");
}
@VisibleForTesting
public String getDefaultNameService() {
return defaultNameService;
}
@VisibleForTesting
public void setDefaultNameService(String defaultNameService) {
this.defaultNameService = defaultNameService;
}
@VisibleForTesting
public boolean isDefaultNSEnable() {
return defaultNSEnable;
}
@VisibleForTesting
public void setDefaultNSEnable(boolean defaultNSRWEnable) {
this.defaultNSEnable = defaultNSRWEnable;
}
}

4
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
View File

@ -42,6 +42,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
"dfs.federation.router.";
public static final String DFS_ROUTER_DEFAULT_NAMESERVICE =
FEDERATION_ROUTER_PREFIX + "default.nameserviceId";
public static final String DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE =
FEDERATION_ROUTER_PREFIX + "default.nameservice.enable";
public static final boolean DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE_DEFAULT =
true;
public static final String DFS_ROUTER_HANDLER_COUNT_KEY =
FEDERATION_ROUTER_PREFIX + "handler.count";
public static final int DFS_ROUTER_HANDLER_COUNT_DEFAULT = 10;

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
View File

@ -1339,7 +1339,7 @@ protected List<RemoteLocation> getLocationsForPath(String path,
this.subclusterResolver.getDestinationForPath(path);
if (location == null) {
throw new IOException("Cannot find locations for " + path + " in " +
this.subclusterResolver);
this.subclusterResolver.getClass().getSimpleName());
}
// We may block some write operations

8
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
View File

@ -31,6 +31,14 @@
</description>
</property>
<property>
<name>dfs.federation.router.default.nameservice.enable</name>
<value>true</value>
<description>
The default subcluster is enabled to read and write files.
</description>
</property>
<property>
<name>dfs.federation.router.rpc.enable</name>
<value>true</value>

26
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
View File

@ -21,6 +21,7 @@
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@ -175,6 +176,31 @@ public void testDestination() throws IOException {
}
@Test
public void testDefaultNameServiceEnable() throws IOException {
assertTrue(mountTable.isDefaultNSEnable());
mountTable.setDefaultNameService("3");
mountTable.removeEntry("/");
assertEquals("3->/unknown",
mountTable.getDestinationForPath("/unknown").toString());
Map<String, String> map = getMountTableEntry("4", "/unknown");
mountTable.addEntry(MountTable.newInstance("/unknown", map));
mountTable.setDefaultNSEnable(false);
assertFalse(mountTable.isDefaultNSEnable());
assertEquals("4->/unknown",
mountTable.getDestinationForPath("/unknown").toString());
try {
mountTable.getDestinationForPath("/");
fail("The getDestinationForPath call should fail.");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
"the default nameservice is disabled to read or write", ioe);
}
}
private void compareLists(List<String> list1, String[] list2) {
assertEquals(list1.size(), list2.length);
for (String item : list2) {