diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 11e7fe8aaf..26ca5d040c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -742,6 +742,10 @@ Release 2.6.0 - UNRELEASED YARN-2724. Skipped uploading a local log file to HDFS if exception is raised when opening it. (Xuan Gong via zjshen) + YARN-1915. Fixed a race condition that client could use the ClientToAMToken + to contact with AM before AM actually receives the ClientToAMTokenMasterKey. + (Jason Lowe via jianhe) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java index 541f7a88c4..29fe9793f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java @@ -36,9 +36,10 @@ @Evolving public class ClientToAMTokenSecretManager extends BaseClientToAMTokenSecretManager { + private static final int MASTER_KEY_WAIT_MSEC = 10 * 1000; // Only one master-key for AM - private SecretKey masterKey; + private volatile SecretKey masterKey; public ClientToAMTokenSecretManager( ApplicationAttemptId applicationAttemptID, byte[] key) { @@ -51,6 +52,23 @@ public ClientToAMTokenSecretManager( } + @Override + public byte[] retrievePassword(ClientToAMTokenIdentifier identifier) + throws InvalidToken { + if (this.masterKey == null) { + synchronized (this) { + while (masterKey == null) { + try { + wait(MASTER_KEY_WAIT_MSEC); + break; + } catch (InterruptedException e) { + } + } + } + } + return super.retrievePassword(identifier); + } + @Override public SecretKey getMasterKey(ApplicationAttemptId applicationAttemptID) { // Only one master-key for AM, just return that. @@ -58,6 +76,9 @@ public SecretKey getMasterKey(ApplicationAttemptId applicationAttemptID) { } public void setMasterKey(byte[] key) { - this.masterKey = SecretManager.createSecretKey(key); + synchronized (this) { + this.masterKey = SecretManager.createSecretKey(key); + notifyAll(); + } } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java index 9b205f66f5..78bc728ac7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java @@ -30,9 +30,12 @@ import java.io.IOException; import java.lang.annotation.Annotation; import java.net.InetSocketAddress; +import java.nio.ByteBuffer; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.Arrays; +import java.util.Timer; +import java.util.TimerTask; import javax.security.sasl.SaslException; @@ -169,6 +172,10 @@ protected void serviceStart() throws Exception { this.address = NetUtils.getConnectAddress(server); super.serviceStart(); } + + public void setClientSecretKey(byte[] key) { + secretMgr.setMasterKey(key); + } } @Test @@ -291,7 +298,7 @@ public RegisterApplicationMasterResponse run() { // Verify for a new version token verifyNewVersionToken(conf, am, token, rm); - + am.stop(); rm.stop(); } @@ -410,4 +417,117 @@ public Void run() throws Exception { } }); } + + @Test(timeout=20000) + public void testClientTokenRace() throws Exception { + + final Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + UserGroupInformation.setConfiguration(conf); + + ContainerManagementProtocol containerManager = + mock(ContainerManagementProtocol.class); + StartContainersResponse mockResponse = mock(StartContainersResponse.class); + when(containerManager.startContainers((StartContainersRequest) any())) + .thenReturn(mockResponse); + final DrainDispatcher dispatcher = new DrainDispatcher(); + + MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, + this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, + getRMContext().getRMDelegationTokenSecretManager()); + }; + + @Override + protected Dispatcher createDispatcher() { + return dispatcher; + } + + @Override + protected void doSecureLogin() throws IOException { + } + }; + rm.start(); + + // Submit an app + RMApp app = rm.submitApp(1024); + + // Set up a node. + MockNM nm1 = rm.registerNode("localhost:1234", 3072); + nm1.nodeHeartbeat(true); + dispatcher.await(); + + nm1.nodeHeartbeat(true); + dispatcher.await(); + + ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId(); + final MockAM mockAM = + new MockAM(rm.getRMContext(), rm.getApplicationMasterService(), + app.getCurrentAppAttempt().getAppAttemptId()); + UserGroupInformation appUgi = + UserGroupInformation.createRemoteUser(appAttempt.toString()); + RegisterApplicationMasterResponse response = + appUgi.doAs(new PrivilegedAction() { + + @Override + public RegisterApplicationMasterResponse run() { + RegisterApplicationMasterResponse response = null; + try { + response = mockAM.registerAppAttempt(); + } catch (Exception e) { + Assert.fail("Exception was not expected"); + } + return response; + } + }); + + // Get the app-report. + GetApplicationReportRequest request = + Records.newRecord(GetApplicationReportRequest.class); + request.setApplicationId(app.getApplicationId()); + GetApplicationReportResponse reportResponse = + rm.getClientRMService().getApplicationReport(request); + ApplicationReport appReport = reportResponse.getApplicationReport(); + org.apache.hadoop.yarn.api.records.Token originalClientToAMToken = + appReport.getClientToAMToken(); + + // ClientToAMToken master key should have been received on register + // application master response. + final ByteBuffer clientMasterKey = response.getClientToAMTokenMasterKey(); + Assert.assertNotNull(clientMasterKey); + Assert.assertTrue(clientMasterKey.array().length > 0); + + // Start the AM with the correct shared-secret. + ApplicationAttemptId appAttemptId = + app.getAppAttempts().keySet().iterator().next(); + Assert.assertNotNull(appAttemptId); + final CustomAM am = new CustomAM(appAttemptId, null); + am.init(conf); + am.start(); + + // Now the real test! + // Set up clients to be able to pick up correct tokens. + SecurityUtil.setSecurityInfoProviders(new CustomSecurityInfo()); + + Token token = + ConverterUtils.convertFromYarn(originalClientToAMToken, am.address); + + // Schedule the key to be set after a significant delay + Timer timer = new Timer(); + TimerTask timerTask = new TimerTask() { + @Override + public void run() { + am.setClientSecretKey(clientMasterKey.array()); + } + }; + timer.schedule(timerTask, 250); + + // connect should pause waiting for the master key to arrive + verifyValidToken(conf, am, token); + + am.stop(); + rm.stop(); + } }