From c5080ef3368cb274915099d7c3185cb2c49faecc Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 11 Dec 2012 19:26:57 +0000 Subject: [PATCH] MAPREDUCE-4861. Cleanup: Remove unused mapreduce.security.token.DelegationTokenRenewal. (kkambatl via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1420345 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../dev-support/findbugs-exclude.xml | 5 - .../token/DelegationTokenRenewal.java | 318 ----------------- .../token/TestDelegationTokenRenewal.java | 330 ------------------ 4 files changed, 3 insertions(+), 653 deletions(-) delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/token/TestDelegationTokenRenewal.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a2c0ebd043..fd9d212cd1 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -208,6 +208,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4800. Cleanup o.a.h.mapred.MapTaskStatus - remove unused code. (kkambatl via tucu) + MAPREDUCE-4861. Cleanup: Remove unused mapreduce.security.token.DelegationTokenRenewal. + (kkambatl via tucu) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml index f548885ff4..08d4c2e7f6 100644 --- a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml +++ b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml @@ -138,11 +138,6 @@ - - - - - diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java deleted file mode 100644 index 9000777069..0000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java +++ /dev/null @@ -1,318 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.security.token; - -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.LinkedBlockingQueue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.util.StringUtils; - - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class DelegationTokenRenewal { - private static final Log LOG = LogFactory.getLog(DelegationTokenRenewal.class); - public static final String SCHEME = "hdfs"; - - /** - * class that is used for keeping tracks of DT to renew - * - */ - private static class DelegationTokenToRenew { - public final Token token; - public final JobID jobId; - public final Configuration conf; - public long expirationDate; - public TimerTask timerTask; - - public DelegationTokenToRenew( - JobID jId, Token t, - Configuration newConf, long newExpirationDate) { - token = t; - jobId = jId; - conf = newConf; - expirationDate = newExpirationDate; - timerTask = null; - if(token==null || jobId==null || conf==null) { - throw new IllegalArgumentException("invalid params for Renew Token" + - ";t="+token+";j="+jobId+";c="+conf); - } - } - public void setTimerTask(TimerTask tTask) { - timerTask = tTask; - } - @Override - public String toString() { - return token + ";exp="+expirationDate; - } - @Override - public boolean equals (Object obj) { - if (obj == this) { - return true; - } else if (obj == null || getClass() != obj.getClass()) { - return false; - } else { - return token.equals(((DelegationTokenToRenew)obj).token); - } - } - @Override - public int hashCode() { - return token.hashCode(); - } - } - - // global single timer (daemon) - private static Timer renewalTimer = new Timer(true); - - //delegation token canceler thread - private static DelegationTokenCancelThread dtCancelThread = - new DelegationTokenCancelThread(); - static { - dtCancelThread.start(); - } - - - //managing the list of tokens using Map - // jobId=>List - private static Set delegationTokens = - Collections.synchronizedSet(new HashSet()); - - private static class DelegationTokenCancelThread extends Thread { - private static class TokenWithConf { - Token token; - Configuration conf; - TokenWithConf(Token token, Configuration conf) { - this.token = token; - this.conf = conf; - } - } - private LinkedBlockingQueue queue = - new LinkedBlockingQueue(); - - public DelegationTokenCancelThread() { - super("Delegation Token Canceler"); - setDaemon(true); - } - public void cancelToken(Token token, - Configuration conf) { - TokenWithConf tokenWithConf = new TokenWithConf(token, conf); - while (!queue.offer(tokenWithConf)) { - LOG.warn("Unable to add token " + token + " for cancellation. " + - "Will retry.."); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - } - - public void run() { - while (true) { - TokenWithConf tokenWithConf = null; - try { - tokenWithConf = queue.take(); - final TokenWithConf current = tokenWithConf; - - if (LOG.isDebugEnabled()) { - LOG.debug("Canceling token " + tokenWithConf.token.getService()); - } - // need to use doAs so that http can find the kerberos tgt - UserGroupInformation.getLoginUser().doAs( - new PrivilegedExceptionAction() { - - @Override - public Void run() throws Exception { - current.token.cancel(current.conf); - return null; - } - }); - } catch (IOException e) { - LOG.warn("Failed to cancel token " + tokenWithConf.token + " " + - StringUtils.stringifyException(e)); - } catch (InterruptedException ie) { - return; - } catch (Throwable t) { - LOG.warn("Got exception " + StringUtils.stringifyException(t) + - ". Exiting.."); - System.exit(-1); - } - } - } - } - //adding token - private static void addTokenToList(DelegationTokenToRenew t) { - delegationTokens.add(t); - } - - public static synchronized void registerDelegationTokensForRenewal( - JobID jobId, Credentials ts, Configuration conf) throws IOException { - if(ts==null) - return; //nothing to add - - Collection > tokens = ts.getAllTokens(); - long now = System.currentTimeMillis(); - - for (Token t : tokens) { - // first renew happens immediately - if (t.isManaged()) { - DelegationTokenToRenew dtr = new DelegationTokenToRenew(jobId, t, conf, - now); - - addTokenToList(dtr); - - setTimerForTokenRenewal(dtr, true); - LOG.info("registering token for renewal for service =" + t.getService() - + " and jobID = " + jobId); - } - } - } - - /** - * Task - to renew a token - * - */ - private static class RenewalTimerTask extends TimerTask { - private DelegationTokenToRenew dttr; - - RenewalTimerTask(DelegationTokenToRenew t) { dttr = t; } - - @Override - public void run() { - Token token = dttr.token; - long newExpirationDate=0; - try { - // need to use doAs so that http can find the kerberos tgt - dttr.expirationDate = UserGroupInformation.getLoginUser().doAs( - new PrivilegedExceptionAction() { - - @Override - public Long run() throws Exception { - return dttr.token.renew(dttr.conf); - } - }); - - if (LOG.isDebugEnabled()) { - LOG.debug("renewing for:" + token.getService() + ";newED=" - + dttr.expirationDate); - } - setTimerForTokenRenewal(dttr, false);// set the next one - } catch (Exception e) { - LOG.error("Exception renewing token" + token + ". Not rescheduled", e); - removeFailedDelegationToken(dttr); - } - } - } - - /** - * find the soonest expiring token and set it for renew - */ - private static void setTimerForTokenRenewal( - DelegationTokenToRenew token, boolean firstTime) { - - // calculate timer time - long now = System.currentTimeMillis(); - long renewIn; - if(firstTime) { - renewIn = now; - } else { - long expiresIn = (token.expirationDate - now); - renewIn = now + expiresIn - expiresIn/10; // little before expiration - } - - // need to create new timer every time - TimerTask tTask = new RenewalTimerTask(token); - token.setTimerTask(tTask); // keep reference to the timer - - renewalTimer.schedule(token.timerTask, new Date(renewIn)); - } - - /** - * removing all tokens renewals - */ - static public void close() { - renewalTimer.cancel(); - delegationTokens.clear(); - } - - // cancel a token - private static void cancelToken(DelegationTokenToRenew t) { - dtCancelThread.cancelToken(t.token, t.conf); - } - - /** - * removing failed DT - * @param jobId - */ - private static void removeFailedDelegationToken(DelegationTokenToRenew t) { - JobID jobId = t.jobId; - if (LOG.isDebugEnabled()) - LOG.debug("removing failed delegation token for jobid=" + jobId + - ";t=" + t.token.getService()); - delegationTokens.remove(t); - // cancel the timer - if(t.timerTask!=null) - t.timerTask.cancel(); - } - - /** - * removing DT for completed jobs - * @param jobId - */ - public static void removeDelegationTokenRenewalForJob(JobID jobId) { - synchronized (delegationTokens) { - Iterator it = delegationTokens.iterator(); - while(it.hasNext()) { - DelegationTokenToRenew dttr = it.next(); - if (dttr.jobId.equals(jobId)) { - if (LOG.isDebugEnabled()) - LOG.debug("removing delegation token for jobid=" + jobId + - ";t=" + dttr.token.getService()); - - // cancel the timer - if(dttr.timerTask!=null) - dttr.timerTask.cancel(); - - // cancel the token - cancelToken(dttr); - - it.remove(); - } - } - } - } -} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/token/TestDelegationTokenRenewal.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/token/TestDelegationTokenRenewal.java deleted file mode 100644 index c2e71e920b..0000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/token/TestDelegationTokenRenewal.java +++ /dev/null @@ -1,330 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.security.token; - - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; -import org.apache.hadoop.security.token.delegation.DelegationKey; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.SecretManager.InvalidToken; -import org.apache.hadoop.security.token.TokenRenewer; -import org.apache.hadoop.util.StringUtils; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -/** - * unit test - - * tests addition/deletion/cancelation of renewals of delegation tokens - * - */ -@Ignore -public class TestDelegationTokenRenewal { - private static final Log LOG = - LogFactory.getLog(TestDelegationTokenRenewal.class); - - private static final Text KIND = - new Text("TestDelegationTokenRenewal.Token"); - - public static class Renewer extends TokenRenewer { - private static int counter = 0; - private static Token lastRenewed = null; - private static Token tokenToRenewIn2Sec = null; - - @Override - public boolean handleKind(Text kind) { - return KIND.equals(kind); - } - - @Override - public boolean isManaged(Token token) throws IOException { - return true; - } - - @Override - public long renew(Token t, Configuration conf) throws IOException { - MyToken token = (MyToken)t; - if(token.isCanceled()) { - throw new InvalidToken("token has been canceled"); - } - lastRenewed = token; - counter ++; - LOG.info("Called MYDFS.renewdelegationtoken " + token + - ";this dfs=" + this.hashCode() + ";c=" + counter); - if(tokenToRenewIn2Sec == token) { - // this token first renewal in 2 seconds - LOG.info("RENEW in 2 seconds"); - tokenToRenewIn2Sec=null; - return 2*1000 + System.currentTimeMillis(); - } else { - return 86400*1000 + System.currentTimeMillis(); - } - } - - @Override - public void cancel(Token t, Configuration conf) { - MyToken token = (MyToken)t; - LOG.info("Cancel token " + token); - token.cancelToken(); - } - - } - - private static Configuration conf; - - @BeforeClass - public static void setUp() throws Exception { - conf = new Configuration(); - - // create a fake FileSystem (MyFS) and assosiate it - // with "hdfs" schema. - URI uri = new URI(DelegationTokenRenewal.SCHEME+"://localhost:0"); - System.out.println("scheme is : " + uri.getScheme()); - conf.setClass("fs." + uri.getScheme() + ".impl", MyFS.class, DistributedFileSystem.class); - FileSystem.setDefaultUri(conf, uri); - LOG.info("filesystem uri = " + FileSystem.getDefaultUri(conf).toString()); - } - - private static class MyDelegationTokenSecretManager extends DelegationTokenSecretManager { - - public MyDelegationTokenSecretManager(long delegationKeyUpdateInterval, - long delegationTokenMaxLifetime, long delegationTokenRenewInterval, - long delegationTokenRemoverScanInterval, FSNamesystem namesystem) { - super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, - delegationTokenRenewInterval, delegationTokenRemoverScanInterval, - namesystem); - } - - @Override //DelegationTokenSecretManager - public void logUpdateMasterKey(DelegationKey key) throws IOException { - return; - } - } - - /** - * add some extra functionality for testing - * 1. toString(); - * 2. cancel() and isCanceled() - */ - private static class MyToken extends Token { - public String status = "GOOD"; - public static final String CANCELED = "CANCELED"; - - public MyToken(DelegationTokenIdentifier dtId1, - MyDelegationTokenSecretManager sm) { - super(dtId1, sm); - setKind(KIND); - status = "GOOD"; - } - - public boolean isCanceled() {return status.equals(CANCELED);} - - public void cancelToken() {this.status=CANCELED;} - - public String toString() { - StringBuilder sb = new StringBuilder(1024); - - sb.append("id="); - String id = StringUtils.byteToHexString(this.getIdentifier()); - int idLen = id.length(); - sb.append(id.substring(idLen-6)); - sb.append(";k="); - sb.append(this.getKind()); - sb.append(";s="); - sb.append(this.getService()); - return sb.toString(); - } - } - - /** - * fake FileSystem - * overwrites three methods - * 1. getDelegationToken() - generates a token - * 2. renewDelegataionToken - counts number of calls, and remembers - * most recently renewed token. - * 3. cancelToken -cancels token (subsequent renew will cause IllegalToken - * exception - */ - static class MyFS extends DistributedFileSystem { - - public MyFS() {} - public void close() {} - @Override - public void initialize(URI uri, Configuration conf) throws IOException {} - - @Override - public MyToken getDelegationToken(Text renewer) throws IOException { - MyToken result = createTokens(renewer); - LOG.info("Called MYDFS.getdelegationtoken " + result); - return result; - } - - } - - /** - * auxilary - create token - * @param renewer - * @return - * @throws IOException - */ - static MyToken createTokens(Text renewer) - throws IOException { - Text user1= new Text("user1"); - - MyDelegationTokenSecretManager sm = new MyDelegationTokenSecretManager( - DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT, - DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT, - DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT, - 3600000, null); - sm.startThreads(); - - DelegationTokenIdentifier dtId1 = - new DelegationTokenIdentifier(user1, renewer, user1); - - MyToken token1 = new MyToken(dtId1, sm); - - - token1.setService(new Text("localhost:0")); - return token1; - } - - - /** - * Basic idea of the test: - * 1. create tokens. - * 2. Mark one of them to be renewed in 2 seconds (istead of - * 24 hourse) - * 3. register them for renewal - * 4. sleep for 3 seconds - * 5. count number of renewals (should 3 initial ones + one extra) - * 6. register another token for 2 seconds - * 7. cancel it immediately - * 8. Sleep and check that the 2 seconds renew didn't happen - * (totally 5 reneals) - * 9. check cancelation - * @throws IOException - * @throws URISyntaxException - */ - @Test - public void testDTRenewal () throws Exception { - MyFS dfs = (MyFS)FileSystem.get(conf); - LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode()); - // Test 1. - add three tokens - make sure exactly one get's renewed - - // get the delegation tokens - MyToken token1, token2, token3; - token1 = dfs.getDelegationToken(new Text("user1")); - token2 = dfs.getDelegationToken(new Text("user2")); - token3 = dfs.getDelegationToken(new Text("user3")); - - //to cause this one to be set for renew in 2 secs - Renewer.tokenToRenewIn2Sec = token1; - LOG.info("token="+token1+" should be renewed for 2 secs"); - - // two distinct Namenodes - String nn1 = DelegationTokenRenewal.SCHEME + "://host1:0"; - String nn2 = DelegationTokenRenewal.SCHEME + "://host2:0"; - String nn3 = DelegationTokenRenewal.SCHEME + "://host3:0"; - - Credentials ts = new Credentials(); - - // register the token for renewal - ts.addToken(new Text(nn1), token1); - ts.addToken(new Text(nn2), token2); - ts.addToken(new Text(nn3), token3); - - // register the tokens for renewal - DelegationTokenRenewal.registerDelegationTokensForRenewal( - new JobID("job1", 1), ts, conf); - // first 3 initial renewals + 1 real - int numberOfExpectedRenewals = 3+1; - - int attempts = 10; - while(attempts-- > 0) { - try { - Thread.sleep(3*1000); // sleep 3 seconds, so it has time to renew - } catch (InterruptedException e) {} - - // since we cannot guarantee timely execution - let's give few chances - if(Renewer.counter==numberOfExpectedRenewals) - break; - } - - assertEquals("renew wasn't called as many times as expected(4):", - numberOfExpectedRenewals, Renewer.counter); - assertEquals("most recently renewed token mismatch", Renewer.lastRenewed, - token1); - - // Test 2. - // add another token ( that expires in 2 secs). Then remove it, before - // time is up. - // Wait for 3 secs , and make sure no renews were called - ts = new Credentials(); - MyToken token4 = dfs.getDelegationToken(new Text("user4")); - - //to cause this one to be set for renew in 2 secs - Renewer.tokenToRenewIn2Sec = token4; - LOG.info("token="+token4+" should be renewed for 2 secs"); - - String nn4 = DelegationTokenRenewal.SCHEME + "://host4:0"; - ts.addToken(new Text(nn4), token4); - - - JobID jid2 = new JobID("job2",1); - DelegationTokenRenewal.registerDelegationTokensForRenewal(jid2, ts, conf); - DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jid2); - numberOfExpectedRenewals = Renewer.counter; // number of renewals so far - try { - Thread.sleep(6*1000); // sleep 6 seconds, so it has time to renew - } catch (InterruptedException e) {} - System.out.println("Counter = " + Renewer.counter + ";t="+ - Renewer.lastRenewed); - - // counter and the token should stil be the old ones - assertEquals("renew wasn't called as many times as expected", - numberOfExpectedRenewals, Renewer.counter); - - // also renewing of the cancelled token should fail - try { - token4.renew(conf); - fail("Renew of canceled token didn't fail"); - } catch (InvalidToken ite) { - //expected - } - } -}