YARN-3472. Fixed possible leak in DelegationTokenRenewer#allTokens. Contributed by Rohith Sharmaks

hadoop-yarn-project/CHANGES.txt

@@ -180,6 +180,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3382. Some of UserMetricsInfo metrics are incorrectly set to root
     queue metrics. (Rohit Agarwal via jianhe)
 
+    YARN-3472. Fixed possible leak in DelegationTokenRenewer#allTokens.
+    (Rohith Sharmaks via jianhe)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java

@@ -577,6 +577,7 @@ private void requestNewHdfsDelegationTokenIfNeeded(
             DelegationTokenToRenew t = iter.next();
             if (t.token.getKind().equals(new Text("HDFS_DELEGATION_TOKEN"))) {
               iter.remove();
+              allTokens.remove(t.token);
               t.cancelTimer();
               LOG.info("Removed expiring token " + t);
             }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java

@@ -906,7 +906,16 @@ protected Token<?>[] obtainSystemTokensForUser(String user,
           new HashMap<ApplicationAccessType, String>(), false, "default", 1,
           credentials);
 
-    // wait for the initial expiring hdfs token to be removed.
+    // wait for the initial expiring hdfs token to be removed from allTokens
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      public Boolean get() {
+        return
+            rm.getRMContext().getDelegationTokenRenewer().getAllTokens()
+            .get(token1) == null;
+      }
+    }, 1000, 20000);
+
+    // wait for the initial expiring hdfs token to be removed from appTokens
     GenericTestUtils.waitFor(new Supplier<Boolean>() {
       public Boolean get() {
         return !rm.getRMContext().getDelegationTokenRenewer()