From 1a95a4524a8c6c7be601ce8b92640a6a76164a2c Mon Sep 17 00:00:00 2001 From: Bibin A Chundatt Date: Wed, 2 May 2018 16:14:28 +0900 Subject: [PATCH] MAPREDUCE-7073. Optimize TokenCache#obtainTokensForNamenodesInternal Signed-off-by: Akira Ajisaka --- .../hadoop/mapreduce/security/TokenCache.java | 14 +++++++++----- .../mapreduce/security/TestTokenCache.java | 18 +++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index 12fced9e0c..1156c6761f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.Set; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -96,8 +97,9 @@ static void obtainTokensForNamenodesInternal(Credentials credentials, for(Path p: ps) { fsSet.add(p.getFileSystem(conf)); } + String masterPrincipal = Master.getMasterPrincipal(conf); for (FileSystem fs : fsSet) { - obtainTokensForNamenodesInternal(fs, credentials, conf); + obtainTokensForNamenodesInternal(fs, credentials, conf, masterPrincipal); } } @@ -122,15 +124,17 @@ static boolean isTokenRenewalExcluded(FileSystem fs, Configuration conf) { * @param conf * @throws IOException */ - static void obtainTokensForNamenodesInternal(FileSystem fs, - Credentials credentials, Configuration conf) throws IOException { + static void obtainTokensForNamenodesInternal(FileSystem fs, + Credentials credentials, Configuration conf, String renewer) + throws IOException { // RM skips renewing token with empty renewer String delegTokenRenewer = ""; if (!isTokenRenewalExcluded(fs, conf)) { - delegTokenRenewer = Master.getMasterPrincipal(conf); - if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { + if (StringUtils.isEmpty(renewer)) { throw new IOException( "Can't get Master Kerberos principal for use as renewer"); + } else { + delegTokenRenewer = renewer; } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java index 127f8ae35b..a44e5331f4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java @@ -56,8 +56,8 @@ public static void setup() throws Exception { @Test public void testObtainTokens() throws Exception { Credentials credentials = new Credentials(); - FileSystem fs = mock(FileSystem.class); - TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf); + FileSystem fs = mock(FileSystem.class); + TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf, renewer); verify(fs).addDelegationTokens(eq(renewer), eq(credentials)); } @@ -105,23 +105,23 @@ private void testBinaryCredentials(boolean hasScheme) throws Exception { checkToken(creds, newerToken1); // get token for fs1, see that fs2's token was loaded - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); checkToken(creds, newerToken1, token2); // get token for fs2, nothing should change since already present - TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); checkToken(creds, newerToken1, token2); // get token for fs3, should only add token for fs3 - TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); Token token3 = creds.getToken(new Text(fs3.getCanonicalServiceName())); assertTrue(token3 != null); checkToken(creds, newerToken1, token2, token3); // be paranoid, check one last time that nothing changes - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); - TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); - TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); checkToken(creds, newerToken1, token2, token3); } @@ -202,7 +202,7 @@ public void testGetTokensForNamenodes() throws IOException, // wait to set, else the obtain tokens call above will fail with FNF conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); String fs_addr = fs1.getCanonicalServiceName(); Token nnt = TokenCache.getDelegationToken(creds, fs_addr); assertNotNull("Token for nn is null", nnt);