big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15545 - Allow WebHdfsFileSystem to read a new delegation token f… (#2255)

Browse Source 
Co-authored-by: Issac Buenrostro <ibuenros@linkedin.com>
This commit is contained in:
Issac Buenrostro 2020-11-12 14:57:26 -08:00 committed by GitHub
parent 5ce18101cb
commit f5e6be337b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 34 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
View File

@ -368,8 +368,8 @@ protected synchronized Token<?> getDelegationToken() throws IOException {
Token<?> token = tokenSelector.selectToken(
new Text(getCanonicalServiceName()), ugi.getTokens());
// ugi tokens are usually indicative of a task which can't
// refetch tokens. even if ugi has credentials, don't attempt
// to get another token to match hdfs/rpc behavior
// refetch tokens. Don't attempt to fetch tokens from the
// namenode in this situation.
if (token != null) {
LOG.debug("Using UGI token: {}", token);
canRefreshDelegationToken = false;
@ -392,6 +392,9 @@ protected synchronized Token<?> getDelegationToken() throws IOException {
@VisibleForTesting
synchronized boolean replaceExpiredDelegationToken() throws IOException {
boolean replaced = false;
if (attemptReplaceDelegationTokenFromUGI()) {
return true;
}
if (canRefreshDelegationToken) {
Token<?> token = getDelegationToken(null);
LOG.debug("Replaced expired token: {}", token);
@ -401,6 +404,17 @@ synchronized boolean replaceExpiredDelegationToken() throws IOException {
return replaced;
}
private synchronized boolean attemptReplaceDelegationTokenFromUGI() {
Token<?> token = tokenSelector.selectToken(
new Text(getCanonicalServiceName()), ugi.getTokens());
if (token != null && !token.equals(delegationToken)) {
LOG.debug("Replaced expired token with new UGI token: {}", token);
setDelegationToken(token);
return true;
}
return false;
}
@Override
protected int getDefaultPort() {
return HdfsClientConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT;

19
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
View File

@ -522,7 +522,8 @@ public WebHdfsFileSystem run() throws IOException {
Assert.assertSame(token, token2);
reset(fs);
// verify an expired ugi token is NOT replaced with a new token
// verify an expired ugi token is not replaced with a new token if the
// ugi does not contain a new token
fs.cancelDelegationToken(token);
for (int i=0; i<2; i++) {
try {
@ -543,6 +544,22 @@ public WebHdfsFileSystem run() throws IOException {
reset(fs);
}
// verify an expired ugi token is replaced with a new token
// if one is available in UGI
token = fs.getDelegationToken(null);
ugi.addToken(token);
reset(fs);
fs.getFileStatus(new Path("/"));
verify(fs, times(2)).getDelegationToken(); // first bad, then good
verify(fs, times(1)).replaceExpiredDelegationToken();
verify(fs, never()).getDelegationToken(anyString());
verify(fs, times(1)).setDelegationToken(eq(token));
token2 = fs.getRenewToken();
Assert.assertNotNull(token2);
Assert.assertEquals(fs.getTokenKind(), token.getKind());
Assert.assertSame(token, token2);
reset(fs);
// verify fs close does NOT cancel the ugi token
fs.close();
verify(fs, never()).getDelegationToken();