big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15096. RBF: GetServerDefaults Should be Cached At Router. Contributed by Ayush Saxena.

Browse Source
This commit is contained in:
Ayush Saxena 2020-01-09 08:26:51 +05:30
parent b1e07d27cc
commit fd30f4c52b
1 changed files with 24 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY;
import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.updateMountPointStatus;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
@ -88,6 +90,7 @@
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -122,6 +125,15 @@ public class RouterClientProtocol implements ClientProtocol {
private final FileSubclusterResolver subclusterResolver;
private final ActiveNamenodeResolver namenodeResolver;
/**
* Caching server defaults so as to prevent redundant calls to namenode,
* similar to DFSClient, caching saves efforts when router connects
* to multiple clients.
*/
private volatile FsServerDefaults serverDefaults;
private volatile long serverDefaultsLastUpdate;
private final long serverDefaultsValidityPeriod;
/** If it requires response from all subclusters. */
private final boolean allowPartialList;
/** Time out when getting the mount statistics. */
@ -155,7 +167,9 @@ public class RouterClientProtocol implements ClientProtocol {
RBFConfigKeys.DFS_ROUTER_CLIENT_MOUNT_TIME_OUT,
RBFConfigKeys.DFS_ROUTER_CLIENT_MOUNT_TIME_OUT_DEFAULT,
TimeUnit.MILLISECONDS);
this.serverDefaultsValidityPeriod = conf.getLong(
DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY,
DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT);
// User and group for reporting
try {
this.superUser = UserGroupInformation.getCurrentUser().getShortUserName();
@ -226,9 +240,15 @@ public LocatedBlocks getBlockLocations(String src, final long offset,
@Override
public FsServerDefaults getServerDefaults() throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
long now = Time.monotonicNow();
if ((serverDefaults == null) || (now - serverDefaultsLastUpdate
> serverDefaultsValidityPeriod)) {
RemoteMethod method = new RemoteMethod("getServerDefaults");
return rpcServer.invokeAtAvailableNs(method, FsServerDefaults.class);
serverDefaults =
rpcServer.invokeAtAvailableNs(method, FsServerDefaults.class);
serverDefaultsLastUpdate = now;
}
return serverDefaults;
}
@Override