big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14687. AuthenticatedURL will reuse bad/expired session cookies. Contributed by Daryn Sharp

Browse Source
This commit is contained in:
Jason Lowe 2017-08-22 16:50:01 -05:00
parent 657dd59cc8
commit c379310212
5 changed files with 403 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
View File

@ -19,8 +19,14 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.CookieHandler;
import java.net.HttpCookie;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -69,14 +75,99 @@ public class AuthenticatedURL {
*/
public static final String AUTH_COOKIE = "hadoop.auth";
private static final String AUTH_COOKIE_EQ = AUTH_COOKIE + "=";
// a lightweight cookie handler that will be attached to url connections.
// client code is not required to extract or inject auth cookies.
private static class AuthCookieHandler extends CookieHandler {
private HttpCookie authCookie;
private Map<String, List<String>> cookieHeaders = Collections.emptyMap();
@Override
public synchronized Map<String, List<String>> get(URI uri,
Map<String, List<String>> requestHeaders) throws IOException {
// call getter so it will reset headers if token is expiring.
getAuthCookie();
return cookieHeaders;
}
@Override
public void put(URI uri, Map<String, List<String>> responseHeaders) {
List<String> headers = responseHeaders.get("Set-Cookie");
if (headers != null) {
for (String header : headers) {
List<HttpCookie> cookies;
try {
cookies = HttpCookie.parse(header);
} catch (IllegalArgumentException iae) {
// don't care. just skip malformed cookie headers.
LOG.debug("Cannot parse cookie header: " + header, iae);
continue;
}
for (HttpCookie cookie : cookies) {
if (AUTH_COOKIE.equals(cookie.getName())) {
setAuthCookie(cookie);
}
}
}
}
}
// return the auth cookie if still valid.
private synchronized HttpCookie getAuthCookie() {
if (authCookie != null && authCookie.hasExpired()) {
setAuthCookie(null);
}
return authCookie;
}
private synchronized void setAuthCookie(HttpCookie cookie) {
final HttpCookie oldCookie = authCookie;
// will redefine if new cookie is valid.
authCookie = null;
cookieHeaders = Collections.emptyMap();
boolean valid = cookie != null && !cookie.getValue().isEmpty() &&
!cookie.hasExpired();
if (valid) {
// decrease lifetime to avoid using a cookie soon to expire.
// allows authenticators to pre-emptively reauthenticate to
// prevent clients unnecessarily receiving a 401.
long maxAge = cookie.getMaxAge();
if (maxAge != -1) {
cookie.setMaxAge(maxAge * 9/10);
valid = !cookie.hasExpired();
}
}
if (valid) {
// v0 cookies value aren't quoted by default but tomcat demands
// quoting.
if (cookie.getVersion() == 0) {
String value = cookie.getValue();
if (!value.startsWith("\"")) {
value = "\"" + value + "\"";
cookie.setValue(value);
}
}
authCookie = cookie;
cookieHeaders = new HashMap<>();
cookieHeaders.put("Cookie", Arrays.asList(cookie.toString()));
}
LOG.trace("Setting token value to {} ({})", authCookie, oldCookie);
}
private void setAuthCookieValue(String value) {
HttpCookie c = null;
if (value != null) {
c = new HttpCookie(AUTH_COOKIE, value);
}
setAuthCookie(c);
}
}
/**
* Client side authentication token.
*/
public static class Token {
private String token;
private final AuthCookieHandler cookieHandler = new AuthCookieHandler();
/**
* Creates a token.
@ -102,7 +193,7 @@ public Token(String tokenStr) {
* @return if a token from the server has been set.
*/
public boolean isSet() {
return token != null;
return cookieHandler.getAuthCookie() != null;
}
/**
@ -111,7 +202,36 @@ public boolean isSet() {
* @param tokenStr string representation of the tokenStr.
*/
void set(String tokenStr) {
token = tokenStr;
cookieHandler.setAuthCookieValue(tokenStr);
}
/**
* Installs a cookie handler for the http request to manage session
* cookies.
* @param url
* @return HttpUrlConnection
* @throws IOException
*/
HttpURLConnection openConnection(URL url,
ConnectionConfigurator connConfigurator) throws IOException {
// the cookie handler is unfortunately a global static. it's a
// synchronized class method so we can safely swap the handler while
// instantiating the connection object to prevent it leaking into
// other connections.
final HttpURLConnection conn;
synchronized(CookieHandler.class) {
CookieHandler current = CookieHandler.getDefault();
CookieHandler.setDefault(cookieHandler);
try {
conn = (HttpURLConnection)url.openConnection();
} finally {
CookieHandler.setDefault(current);
}
}
if (connConfigurator != null) {
connConfigurator.configure(conn);
}
return conn;
}
/**
@ -121,7 +241,15 @@ void set(String tokenStr) {
*/
@Override
public String toString() {
return token;
String value = "";
HttpCookie authCookie = cookieHandler.getAuthCookie();
if (authCookie != null) {
value = authCookie.getValue();
if (value.startsWith("\"")) { // tests don't want the quotes.
value = value.substring(1, value.length()-1);
}
}
return value;
}
}
@ -218,27 +346,25 @@ public HttpURLConnection openConnection(URL url, Token token) throws IOException
throw new IllegalArgumentException("token cannot be NULL");
}
authenticator.authenticate(url, token);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
if (connConfigurator != null) {
conn = connConfigurator.configure(conn);
}
injectToken(conn, token);
return conn;
// allow the token to create the connection with a cookie handler for
// managing session cookies.
return token.openConnection(url, connConfigurator);
}
/**
* Helper method that injects an authentication token to send with a connection.
* Helper method that injects an authentication token to send with a
* connection. Callers should prefer using
* {@link Token#openConnection(URL, ConnectionConfigurator)} which
* automatically manages authentication tokens.
*
* @param conn connection to inject the authentication token into.
* @param token authentication token to inject.
*/
public static void injectToken(HttpURLConnection conn, Token token) {
String t = token.token;
if (t != null) {
if (!t.startsWith("\"")) {
t = "\"" + t + "\"";
}
conn.addRequestProperty("Cookie", AUTH_COOKIE_EQ + t);
HttpCookie authCookie = token.cookieHandler.getAuthCookie();
if (authCookie != null) {
conn.addRequestProperty("Cookie", authCookie.toString());
}
}
@ -258,24 +384,10 @@ public static void extractToken(HttpURLConnection conn, Token token) throws IOEx
if (respCode == HttpURLConnection.HTTP_OK
|| respCode == HttpURLConnection.HTTP_CREATED
|| respCode == HttpURLConnection.HTTP_ACCEPTED) {
Map<String, List<String>> headers = conn.getHeaderFields();
List<String> cookies = headers.get("Set-Cookie");
if (cookies != null) {
for (String cookie : cookies) {
if (cookie.startsWith(AUTH_COOKIE_EQ)) {
String value = cookie.substring(AUTH_COOKIE_EQ.length());
int separator = value.indexOf(";");
if (separator > -1) {
value = value.substring(0, separator);
}
if (value.length() > 0) {
LOG.trace("Setting token value to {} ({}), resp={}", value,
token, respCode);
token.set(value);
}
}
}
}
// cookie handler should have already extracted the token. try again
// for backwards compatibility if this method is called on a connection
// not opened via this instance.
token.cookieHandler.put(null, conn.getHeaderFields());
} else if (respCode == HttpURLConnection.HTTP_NOT_FOUND) {
LOG.trace("Setting token value to null ({}), resp={}", token, respCode);
token.set(null);

30
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
View File

@ -147,7 +147,6 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
}
private URL url;
private HttpURLConnection conn;
private Base64 base64;
private ConnectionConfigurator connConfigurator;
@ -182,10 +181,7 @@ public void authenticate(URL url, AuthenticatedURL.Token token)
if (!token.isSet()) {
this.url = url;
base64 = new Base64(0);
conn = (HttpURLConnection) url.openConnection();
if (connConfigurator != null) {
conn = connConfigurator.configure(conn);
}
HttpURLConnection conn = token.openConnection(url, connConfigurator);
conn.setRequestMethod(AUTH_HTTP_METHOD);
conn.connect();
@ -200,7 +196,7 @@ public void authenticate(URL url, AuthenticatedURL.Token token)
}
needFallback = true;
}
if (!needFallback && isNegotiate()) {
if (!needFallback && isNegotiate(conn)) {
LOG.debug("Performing our own SPNEGO sequence.");
doSpnegoSequence(token);
} else {
@ -249,7 +245,7 @@ private boolean isTokenKerberos(AuthenticatedURL.Token token)
/*
* Indicates if the response is starting a SPNEGO negotiation.
*/
private boolean isNegotiate() throws IOException {
private boolean isNegotiate(HttpURLConnection conn) throws IOException {
boolean negotiate = false;
if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
@ -267,7 +263,8 @@ private boolean isNegotiate() throws IOException {
* @throws IOException if an IO error occurred.
* @throws AuthenticationException if an authentication error occurred.
*/
private void doSpnegoSequence(AuthenticatedURL.Token token) throws IOException, AuthenticationException {
private void doSpnegoSequence(final AuthenticatedURL.Token token)
throws IOException, AuthenticationException {
try {
AccessControlContext context = AccessController.getContext();
Subject subject = Subject.getSubject(context);
@ -308,13 +305,15 @@ public Void run() throws Exception {
// Loop while the context is still not established
while (!established) {
HttpURLConnection conn =
token.openConnection(url, connConfigurator);
outToken = gssContext.initSecContext(inToken, 0, inToken.length);
if (outToken != null) {
sendToken(outToken);
sendToken(conn, outToken);
}
if (!gssContext.isEstablished()) {
inToken = readToken();
inToken = readToken(conn);
} else {
established = true;
}
@ -337,18 +336,14 @@ public Void run() throws Exception {
} catch (LoginException ex) {
throw new AuthenticationException(ex);
}
AuthenticatedURL.extractToken(conn, token);
}
/*
* Sends the Kerberos token to the server.
*/
private void sendToken(byte[] outToken) throws IOException {
private void sendToken(HttpURLConnection conn, byte[] outToken)
throws IOException {
String token = base64.encodeToString(outToken);
conn = (HttpURLConnection) url.openConnection();
if (connConfigurator != null) {
conn = connConfigurator.configure(conn);
}
conn.setRequestMethod(AUTH_HTTP_METHOD);
conn.setRequestProperty(AUTHORIZATION, NEGOTIATE + " " + token);
conn.connect();
@ -357,7 +352,8 @@ private void sendToken(byte[] outToken) throws IOException {
/*
* Retrieves the Kerberos token returned by the server.
*/
private byte[] readToken() throws IOException, AuthenticationException {
private byte[] readToken(HttpURLConnection conn)
throws IOException, AuthenticationException {
int status = conn.getResponseCode();
if (status == HttpURLConnection.HTTP_OK || status == HttpURLConnection.HTTP_UNAUTHORIZED) {
String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);

5
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
View File

@ -68,10 +68,7 @@ public void authenticate(URL url, AuthenticatedURL.Token token) throws IOExcepti
String paramSeparator = (strUrl.contains("?")) ? "&" : "?";
strUrl += paramSeparator + USER_NAME_EQ + getUserName();
url = new URL(strUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
if (connConfigurator != null) {
conn = connConfigurator.configure(conn);
}
HttpURLConnection conn = token.openConnection(url, connConfigurator);
conn.setRequestMethod("OPTIONS");
conn.connect();
AuthenticatedURL.extractToken(conn, token);

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
View File

@ -32,8 +32,6 @@
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
@ -522,17 +520,6 @@ private <T> T call(HttpURLConnection conn, Object jsonOutput,
authRetryCount - 1);
}
}
try {
AuthenticatedURL.extractToken(conn, authToken);
if (LOG.isDebugEnabled()) {
LOG.debug("Extracted token, authToken={}, its dt={}", authToken,
authToken.getDelegationToken());
}
} catch (AuthenticationException e) {
// Ignore the AuthExceptions.. since we are just using the method to
// extract and set the authToken.. (Workaround till we actually fix
// AuthenticatedURL properly to set authToken post initialization)
}
HttpExceptionUtils.validateResponse(conn, expectedResponse);
if (conn.getContentType() != null
&& conn.getContentType().trim().toLowerCase()

242
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWithSpengo.java
View File

@ -22,9 +22,12 @@
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.KerberosTestUtils;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.apache.hadoop.security.authentication.util.Signer;
@ -32,6 +35,7 @@
import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.ietf.jgss.GSSException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@ -45,7 +49,14 @@
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.security.AccessController;
import java.security.PrivilegedExceptionAction;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
import javax.security.auth.Subject;
import javax.servlet.ServletContext;
import static org.junit.Assert.assertTrue;
/**
@ -72,16 +83,25 @@ public class TestHttpServerWithSpengo {
private static MiniKdc testMiniKDC;
private static File secretFile = new File(testRootDir, SECRET_STR);
private static UserGroupInformation authUgi;
@BeforeClass
public static void setUp() throws Exception {
try {
testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir);
testMiniKDC.start();
testMiniKDC.createPrincipal(
httpSpnegoKeytabFile, HTTP_USER + "/localhost");
httpSpnegoKeytabFile, HTTP_USER + "/localhost", "keytab-user");
} catch (Exception e) {
assertTrue("Couldn't setup MiniKDC", false);
}
System.setProperty("sun.security.krb5.debug", "true");
Configuration conf = new Configuration();
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
UserGroupInformation.setConfiguration(conf);
authUgi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(
"keytab-user", httpSpnegoKeytabFile.toString());
Writer w = new FileWriter(secretFile);
w.write("secret");
w.close();
@ -209,6 +229,226 @@ public void testAuthenticationWithProxyUser() throws Exception {
}
}
@Test
public void testSessionCookie() throws Exception {
Configuration conf = new Configuration();
conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
AuthenticationFilterInitializer.class.getName());
conf.set(PREFIX + "type", "kerberos");
conf.setBoolean(PREFIX + "simple.anonymous.allowed", false);
conf.set(PREFIX + "signer.secret.provider",
TestSignerSecretProvider.class.getName());
conf.set(PREFIX + "kerberos.keytab",
httpSpnegoKeytabFile.getAbsolutePath());
conf.set(PREFIX + "kerberos.principal", httpSpnegoPrincipal);
conf.set(PREFIX + "cookie.domain", realm);
conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
true);
//setup logs dir
System.setProperty("hadoop.log.dir", testRootDir.getAbsolutePath());
HttpServer2 httpServer = null;
// Create http server to test.
httpServer = getCommonBuilder()
.setConf(conf)
.build();
httpServer.start();
// Get signer to encrypt token
final Signer signer = new Signer(new TestSignerSecretProvider());
final AuthenticatedURL authUrl = new AuthenticatedURL();
final URL url = new URL("http://" + NetUtils.getHostPortString(
httpServer.getConnectorAddress(0)) + "/conf");
// this illustrates an inconsistency with AuthenticatedURL. the
// authenticator is only called when the token is not set. if the
// authenticator fails then it must throw an AuthenticationException to
// the caller, yet the caller may see 401 for subsequent requests
// that require re-authentication like token expiration.
final UserGroupInformation simpleUgi =
UserGroupInformation.createRemoteUser("simple-user");
authUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
TestSignerSecretProvider.rollSecret();
HttpURLConnection conn = null;
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
// initial request should trigger authentication and set the token.
conn = authUrl.openConnection(url, token);
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
Assert.assertTrue(token.isSet());
String cookie = token.toString();
// token should not change.
conn = authUrl.openConnection(url, token);
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
Assert.assertTrue(token.isSet());
Assert.assertEquals(cookie, token.toString());
// roll secret to invalidate token.
TestSignerSecretProvider.rollSecret();
conn = authUrl.openConnection(url, token);
// this may or may not happen. under normal circumstances the
// jdk will silently renegotiate and the client never sees a 401.
// however in some cases the jdk will give up doing spnego. since
// the token is already set, the authenticator isn't invoked (which
// would do the spnego if the jdk doesn't), which causes the client
// to see a 401.
if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
// if this happens, the token should be cleared which means the
// next request should succeed and receive a new token.
Assert.assertFalse(token.isSet());
conn = authUrl.openConnection(url, token);
}
// token should change.
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
Assert.assertTrue(token.isSet());
Assert.assertNotEquals(cookie, token.toString());
cookie = token.toString();
// token should not change.
for (int i=0; i < 3; i++) {
conn = authUrl.openConnection(url, token);
Assert.assertEquals("attempt"+i,
HttpURLConnection.HTTP_OK, conn.getResponseCode());
Assert.assertTrue(token.isSet());
Assert.assertEquals(cookie, token.toString());
}
// blow out the kerberos creds test only auth token is used.
Subject s = Subject.getSubject(AccessController.getContext());
Set<Object> oldCreds = new HashSet<>(s.getPrivateCredentials());
s.getPrivateCredentials().clear();
// token should not change.
for (int i=0; i < 3; i++) {
try {
conn = authUrl.openConnection(url, token);
Assert.assertEquals("attempt"+i,
HttpURLConnection.HTTP_OK, conn.getResponseCode());
} catch (AuthenticationException ae) {
Assert.fail("attempt"+i+" "+ae);
}
Assert.assertTrue(token.isSet());
Assert.assertEquals(cookie, token.toString());
}
// invalidate token. connections should fail now and token should be
// unset.
TestSignerSecretProvider.rollSecret();
conn = authUrl.openConnection(url, token);
Assert.assertEquals(
HttpURLConnection.HTTP_UNAUTHORIZED, conn.getResponseCode());
Assert.assertFalse(token.isSet());
Assert.assertEquals("", token.toString());
// restore the kerberos creds, should work again.
s.getPrivateCredentials().addAll(oldCreds);
conn = authUrl.openConnection(url, token);
Assert.assertEquals(
HttpURLConnection.HTTP_OK, conn.getResponseCode());
Assert.assertTrue(token.isSet());
cookie = token.toString();
// token should not change.
for (int i=0; i < 3; i++) {
conn = authUrl.openConnection(url, token);
Assert.assertEquals("attempt"+i,
HttpURLConnection.HTTP_OK, conn.getResponseCode());
Assert.assertTrue(token.isSet());
Assert.assertEquals(cookie, token.toString());
}
return null;
}
});
simpleUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
TestSignerSecretProvider.rollSecret();
AuthenticatedURL authUrl = new AuthenticatedURL();
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
HttpURLConnection conn = null;
// initial connect with unset token will trigger authenticator which
// should fail since we have no creds and leave token unset.
try {
authUrl.openConnection(url, token);
Assert.fail("should fail with no credentials");
} catch (AuthenticationException ae) {
Assert.assertNotNull(ae.getCause());
Assert.assertEquals(GSSException.class, ae.getCause().getClass());
GSSException gsse = (GSSException)ae.getCause();
Assert.assertEquals(GSSException.NO_CRED, gsse.getMajor());
} catch (Throwable t) {
Assert.fail("Unexpected exception" + t);
}
Assert.assertFalse(token.isSet());
// create a valid token and save its value.
token = getEncryptedAuthToken(signer, "valid");
String cookie = token.toString();
// server should accept token. after the request the token should
// be set to the same value (ie. server didn't reissue cookie)
conn = authUrl.openConnection(url, token);
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
Assert.assertTrue(token.isSet());
Assert.assertEquals(cookie, token.toString());
conn = authUrl.openConnection(url, token);
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
Assert.assertTrue(token.isSet());
Assert.assertEquals(cookie, token.toString());
// change the secret to effectively invalidate the cookie. see above
// regarding inconsistency. the authenticator has no way to know the
// token is bad, so the client will encounter a 401 instead of
// AuthenticationException.
TestSignerSecretProvider.rollSecret();
conn = authUrl.openConnection(url, token);
Assert.assertEquals(
HttpURLConnection.HTTP_UNAUTHORIZED, conn.getResponseCode());
Assert.assertFalse(token.isSet());
Assert.assertEquals("", token.toString());
return null;
}
});
}
public static class TestSignerSecretProvider extends SignerSecretProvider {
static int n = 0;
static byte[] secret;
static void rollSecret() {
secret = ("secret[" + (n++) + "]").getBytes();
}
public TestSignerSecretProvider() {
}
@Override
public void init(Properties config, ServletContext servletContext,
long tokenValidity) throws Exception {
rollSecret();
}
@Override
public byte[] getCurrentSecret() {
return secret;
}
@Override
public byte[][] getAllSecrets() {
return new byte[][]{secret};
}
}
private AuthenticatedURL.Token getEncryptedAuthToken(Signer signer,
String user) throws Exception {