HDFS-7597. DelegationTokenIdentifier should cache the TokenIdentifier to UGI mapping. Contributed by Daryn Sharp, Bob Hansen, and Xiao Chen.

This commit is contained in:
Akira Ajisaka 2016-06-22 11:00:16 +09:00
parent d8107fcd1c
commit d433b16ce6
5 changed files with 71 additions and 2 deletions

View File

@ -21,13 +21,20 @@
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import org.apache.commons.collections.map.LRUMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import com.google.common.annotations.VisibleForTesting;
/**
* A delegation token identifier that is specific to HDFS.
*/
@ -37,6 +44,15 @@ public class DelegationTokenIdentifier
public static final Text HDFS_DELEGATION_KIND =
new Text("HDFS_DELEGATION_TOKEN");
@SuppressWarnings("unchecked")
private static Map<TokenIdentifier, UserGroupInformation> ugiCache =
Collections.synchronizedMap(new LRUMap(64));
@VisibleForTesting
public void clearCache() {
ugiCache.clear();
}
/**
* Create an empty delegation token identifier for reading into.
*/
@ -58,6 +74,16 @@ public Text getKind() {
return HDFS_DELEGATION_KIND;
}
@Override
public UserGroupInformation getUser() {
UserGroupInformation ugi = ugiCache.get(this);
if (ugi == null) {
ugi = super.getUser();
ugiCache.put(this, ugi);
}
return ugi;
}
@Override
public String toString() {
StringBuilder sbld = new StringBuilder();

View File

@ -61,6 +61,13 @@ public static synchronized void init(Configuration conf) {
}
}
@VisibleForTesting
void clearCache() throws IOException {
if (UserGroupInformation.isSecurityEnabled()) {
params.delegationToken().decodeIdentifier().clearCache();
}
}
UserGroupInformation ugi() throws IOException {
UserGroupInformation ugi;

View File

@ -239,7 +239,36 @@ public Object run() throws IOException {
}
});
}
@Test
public void testDelegationTokenUgi() throws Exception {
final DistributedFileSystem dfs = cluster.getFileSystem();
Token<?>[] tokens = dfs.addDelegationTokens("renewer", null);
Assert.assertEquals(1, tokens.length);
Token<?> token1 = tokens[0];
DelegationTokenIdentifier ident =
(DelegationTokenIdentifier) token1.decodeIdentifier();
UserGroupInformation expectedUgi = ident.getUser();
// get 2 new instances (clones) of the identifier, query their ugi
// twice each, all ugi instances should be equivalent
for (int i=0; i<2; i++) {
DelegationTokenIdentifier identClone =
(DelegationTokenIdentifier)token1.decodeIdentifier();
Assert.assertEquals(ident, identClone);
Assert.assertNotSame(ident, identClone);
Assert.assertSame(expectedUgi, identClone.getUser());
Assert.assertSame(expectedUgi, identClone.getUser());
}
// a new token must decode to a different ugi instance than the first token
tokens = dfs.addDelegationTokens("renewer", null);
Assert.assertEquals(1, tokens.length);
Token<?> token2 = tokens[0];
Assert.assertNotEquals(token1, token2);
Assert.assertNotSame(expectedUgi, token2.decodeIdentifier().getUser());
}
/**
* Test that the delegation token secret manager only runs when the
* NN is out of safe mode. This is because the secret manager

View File

@ -109,6 +109,7 @@ public void testGetUgi() throws IOException {
//Test attribute name.node.address
//Set the nnaddr url parameter to null.
token.decodeIdentifier().clearCache();
when(request.getParameter(JspHelper.NAMENODE_ADDRESS)).thenReturn(null);
InetSocketAddress addr = new InetSocketAddress("localhost", 2222);
when(context.getAttribute(NameNodeHttpServer.NAMENODE_ADDRESS_ATTRIBUTE_KEY))
@ -116,7 +117,12 @@ public void testGetUgi() throws IOException {
verifyServiceInToken(context, request, addr.getAddress().getHostAddress()
+ ":2222");
//Test service already set in the token
//Test service already set in the token and DN doesn't change service
//when it doesn't know the NN service addr
userText = new Text(user+"2");
dtId = new DelegationTokenIdentifier(userText, userText, null);
token = new Token<DelegationTokenIdentifier>(
dtId, new DummySecretManager(0, 0, 0, 0));
token.setService(new Text("3.3.3.3:3333"));
tokenString = token.encodeToUrlString();
//Set the name.node.address attribute in Servlet context to null

View File

@ -121,6 +121,7 @@ public void testUGICacheSecure() throws Exception {
"With UGI cache, two UGIs for the different token should not be same",
ugi11, url22);
ugiProvider2.clearCache();
awaitCacheEmptyDueToExpiration();
ugi12 = ugiProvider1.ugi();
url22 = ugiProvider2.ugi();