big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

YARN-5092. TestRMDelegationTokens fails intermittently. Contributed by Jason Lowe.

Browse Source
This commit is contained in:
Rohith Sharma K S 2016-07-21 12:47:27 +05:30
parent be34b2a8fd
commit 557a245d83
1 changed files with 22 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
View File

@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityM
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
@ -54,23 +53,24 @@ import org.junit.Test;
public class TestRMDelegationTokens { public class TestRMDelegationTokens {
private YarnConfiguration conf; private YarnConfiguration testConf;
@Before @Before
public void setup() { public void setup() {
Logger rootLogger = LogManager.getRootLogger(); Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG); rootLogger.setLevel(Level.DEBUG);
ExitUtil.disableSystemExit(); ExitUtil.disableSystemExit();
conf = new YarnConfiguration(); testConf = new YarnConfiguration();
UserGroupInformation.setConfiguration(conf); UserGroupInformation.setLoginUser(null);
conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); UserGroupInformation.setConfiguration(testConf);
conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); testConf.set(YarnConfiguration.RM_STORE,
MemoryRMStateStore.class.getName());
} }
// Test the DT mast key in the state-store when the mast key is being rolled. // Test the DT mast key in the state-store when the mast key is being rolled.
@Test(timeout = 15000) @Test(timeout = 15000)
public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception { public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration(testConf);
conf.set("hadoop.security.authentication", "kerberos"); conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf); UserGroupInformation.setConfiguration(conf);
MemoryRMStateStore memStore = new MemoryRMStateStore(); MemoryRMStateStore memStore = new MemoryRMStateStore();
@ -93,9 +93,6 @@ public class TestRMDelegationTokens {
rm1.getRMContext().getRMDelegationTokenSecretManager(); rm1.getRMContext().getRMDelegationTokenSecretManager();
// assert all master keys are saved // assert all master keys are saved
Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState); Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState);
Set<DelegationKey> expiringKeys = new HashSet<DelegationKey>();
expiringKeys.addAll(dtSecretManager.getAllMasterKeys());
// request to generate a RMDelegationToken // request to generate a RMDelegationToken
GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class); GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class);
@ -131,13 +128,13 @@ public class TestRMDelegationTokens {
@Test(timeout = 15000) @Test(timeout = 15000)
public void testRemoveExpiredMasterKeyInRMStateStore() throws Exception { public void testRemoveExpiredMasterKeyInRMStateStore() throws Exception {
MemoryRMStateStore memStore = new MemoryRMStateStore(); MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf); memStore.init(testConf);
RMState rmState = memStore.getState(); RMState rmState = memStore.getState();
Set<DelegationKey> rmDTMasterKeyState = Set<DelegationKey> rmDTMasterKeyState =
rmState.getRMDTSecretManagerState().getMasterKeyState(); rmState.getRMDTSecretManagerState().getMasterKeyState();
MockRM rm1 = new MyMockRM(conf, memStore); MockRM rm1 = new MyMockRM(testConf, memStore);
rm1.start(); rm1.start();
RMDelegationTokenSecretManager dtSecretManager = RMDelegationTokenSecretManager dtSecretManager =
rm1.getRMContext().getRMDelegationTokenSecretManager(); rm1.getRMContext().getRMDelegationTokenSecretManager();
@ -159,6 +156,7 @@ public class TestRMDelegationTokens {
break; break;
Thread.sleep(500); Thread.sleep(500);
} }
rm1.stop();
} }
class MyMockRM extends TestSecurityMockRM { class MyMockRM extends TestSecurityMockRM {
@ -169,7 +167,7 @@ public class TestRMDelegationTokens {
@Override @Override
protected RMSecretManagerService createRMSecretManagerService() { protected RMSecretManagerService createRMSecretManagerService() {
return new RMSecretManagerService(conf, rmContext) { return new RMSecretManagerService(testConf, rmContext) {
@Override @Override
protected RMDelegationTokenSecretManager protected RMDelegationTokenSecretManager
@ -208,7 +206,17 @@ public class TestRMDelegationTokens {
for (int keyId : allKeys.keySet()) { for (int keyId : allKeys.keySet()) {
if (keyId == currentId) { if (keyId == currentId) {
DelegationKey currentKey = allKeys.get(keyId); DelegationKey currentKey = allKeys.get(keyId);
Assert.assertTrue(rmDTMasterKeyState.contains(currentKey)); // There's a small window where the key expiry has changed in memory
// but not the state store yet, and DelegationKey hashcode/equals
// uses the expiry so the contains method will fail to find it.
boolean found = false;
for (DelegationKey k : rmDTMasterKeyState) {
if (k.getKeyId() == keyId) {
found = true;
break;
}
}
Assert.assertTrue(found);
return currentKey; return currentKey;
} }
} }