big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5098. Fixed ResourceManager's DelegationTokenRenewer to replace expiring system-tokens if RM stops and only restarts after a long time. Contributed by Jian He.

Browse Source
This commit is contained in:
Vinod Kumar Vavilapalli 2016-06-03 13:00:07 -07:00
parent 99cc439e29
commit f10ebc67f5
2 changed files with 118 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
View File

@ -53,6 +53,7 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.service.AbstractService;
@ -459,6 +460,18 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt)
try {
renewToken(dttr);
} catch (IOException ioe) {
if (ioe instanceof SecretManager.InvalidToken
&& dttr.maxDate < Time.now()
&& evt instanceof DelegationTokenRenewerAppRecoverEvent
&& token.getKind().equals(HDFS_DELEGATION_KIND)) {
LOG.info("Failed to renew hdfs token " + dttr
+ " on recovery as it expired, requesting new hdfs token for "
+ applicationId + ", user=" + evt.getUser(), ioe);
requestNewHdfsDelegationTokenAsProxyUser(
Arrays.asList(applicationId), evt.getUser(),
evt.shouldCancelAtEnd());
continue;
}
throw new IOException("Failed to renew token: " + dttr.token, ioe);
}
}
@ -485,7 +498,8 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt)
}
if (!hasHdfsToken) {
requestNewHdfsDelegationToken(Arrays.asList(applicationId), evt.getUser(),
requestNewHdfsDelegationTokenAsProxyUser(Arrays.asList(applicationId),
evt.getUser(),
shouldCancelAtEnd);
}
}
@ -586,8 +600,7 @@ public Long run() throws Exception {
} catch (InterruptedException e) {
throw new IOException(e);
}
LOG.info("Renewed delegation-token= [" + dttr + "], for "
+ dttr.referringAppIds);
LOG.info("Renewed delegation-token= [" + dttr + "]");
}
// Request new hdfs token if the token is about to expire, and remove the old
@ -625,12 +638,12 @@ private void requestNewHdfsDelegationTokenIfNeeded(
}
}
LOG.info("Token= (" + dttr + ") is expiring, request new token.");
requestNewHdfsDelegationToken(applicationIds, dttr.user,
requestNewHdfsDelegationTokenAsProxyUser(applicationIds, dttr.user,
dttr.shouldCancelAtEnd);
}
}
private void requestNewHdfsDelegationToken(
private void requestNewHdfsDelegationTokenAsProxyUser(
Collection<ApplicationId> referringAppIds,
String user, boolean shouldCancelAtEnd) throws IOException,
InterruptedException {
@ -912,8 +925,8 @@ private void handleDTRenewerAppRecoverEvent(
// Setup tokens for renewal during recovery
DelegationTokenRenewer.this.handleAppSubmitEvent(event);
} catch (Throwable t) {
LOG.warn(
"Unable to add the application to the delegation token renewer.", t);
LOG.warn("Unable to add the application to the delegation token"
+ " renewer on recovery.", t);
}
}

98
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
View File

@ -43,6 +43,7 @@
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@ -84,6 +85,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityMockRM;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@ -968,6 +970,101 @@ public Boolean get() {
Assert.assertTrue(appCredentials.getAllTokens().contains(expectedToken));
}
// 1. token is expired before app completes.
// 2. RM shutdown.
// 3. When RM recovers the app, token renewal will fail as token expired.
// RM should request a new token and sent it to NM for log-aggregation.
@Test
public void testRMRestartWithExpiredToken() throws Exception {
Configuration yarnConf = new YarnConfiguration();
yarnConf
.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
yarnConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
yarnConf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
yarnConf
.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
UserGroupInformation.setConfiguration(yarnConf);
// create Token1:
Text userText1 = new Text("user1");
DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(userText1,
new Text("renewer1"), userText1);
final Token<DelegationTokenIdentifier> originalToken =
new Token<>(dtId1.getBytes(), "password1".getBytes(), dtId1.getKind(),
new Text("service1"));
Credentials credentials = new Credentials();
credentials.addToken(userText1, originalToken);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(yarnConf);
MockRM rm1 = new TestSecurityMockRM(yarnConf, memStore);
rm1.start();
RMApp app = rm1.submitApp(200, "name", "user",
new HashMap<ApplicationAccessType, String>(), false, "default", 1,
credentials);
// create token2
Text userText2 = new Text("user1");
DelegationTokenIdentifier dtId2 =
new DelegationTokenIdentifier(userText1, new Text("renewer2"),
userText2);
final Token<DelegationTokenIdentifier> updatedToken =
new Token<DelegationTokenIdentifier>(dtId2.getBytes(),
"password2".getBytes(), dtId2.getKind(), new Text("service2"));
AtomicBoolean firstRenewInvoked = new AtomicBoolean(false);
AtomicBoolean secondRenewInvoked = new AtomicBoolean(false);
MockRM rm2 = new TestSecurityMockRM(yarnConf, memStore) {
@Override
protected DelegationTokenRenewer createDelegationTokenRenewer() {
return new DelegationTokenRenewer() {
@Override
protected void renewToken(final DelegationTokenToRenew dttr)
throws IOException {
if (dttr.token.equals(updatedToken)) {
secondRenewInvoked.set(true);
super.renewToken(dttr);
} else if (dttr.token.equals(originalToken)){
firstRenewInvoked.set(true);
throw new InvalidToken("Failed to renew");
} else {
throw new IOException("Unexpected");
}
}
@Override
protected Token<?>[] obtainSystemTokensForUser(String user,
final Credentials credentials) throws IOException {
credentials.addToken(updatedToken.getService(), updatedToken);
return new Token<?>[] { updatedToken };
}
};
}
};
// simulating restart the rm
rm2.start();
// check nm can retrieve the token
final MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService());
nm1.registerNode();
NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
ByteBuffer tokenBuffer =
response.getSystemCredentialsForApps().get(app.getApplicationId());
Assert.assertNotNull(tokenBuffer);
Credentials appCredentials = new Credentials();
DataInputByteBuffer buf = new DataInputByteBuffer();
tokenBuffer.rewind();
buf.reset(tokenBuffer);
appCredentials.readTokenStorageStream(buf);
Assert.assertTrue(firstRenewInvoked.get() && secondRenewInvoked.get());
Assert.assertTrue(appCredentials.getAllTokens().contains(updatedToken));
}
// YARN will get the token for the app submitted without the delegation token.
@Test
public void testAppSubmissionWithoutDelegationToken() throws Exception {
@ -1158,4 +1255,5 @@ public void testCancelWithMultipleAppSubmissions() throws Exception{
Assert.assertTrue(dttr.isTimerCancelled());
Assert.assertTrue(Renewer.cancelled);
}
}