big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12052. Set SWEBHDFS delegation token kind when ssl is enabled in HttpFS. Contributed by Zoran Dimitrijevic.

Browse Source
This commit is contained in:
John Zhuge 2017-07-11 11:19:08 -07:00
parent 3a7f02b815
commit 12c8fdceaf
4 changed files with 154 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java
View File

@ -95,8 +95,12 @@ protected Properties getConfiguration(String configPrefix,
throw new RuntimeException("Could not read HttpFS signature secret file: " + signatureSecretFile);
}
setAuthHandlerClass(props);
String dtkind = WebHdfsConstants.WEBHDFS_TOKEN_KIND.toString();
if (conf.getBoolean(HttpFSServerWebServer.SSL_ENABLED_KEY, false)) {
dtkind = WebHdfsConstants.SWEBHDFS_TOKEN_KIND.toString();
}
props.setProperty(KerberosDelegationTokenAuthenticationHandler.TOKEN_KIND,
WebHdfsConstants.WEBHDFS_TOKEN_KIND.toString());
dtkind);
return props;
}

2
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebServer.java
View File

@ -52,7 +52,7 @@ public class HttpFSServerWebServer {
private static final String HTTP_HOST_DEFAULT = "0.0.0.0";
// SSL properties
private static final String SSL_ENABLED_KEY = "hadoop.httpfs.ssl.enabled";
static final String SSL_ENABLED_KEY = "httpfs.ssl.enabled";
private static final boolean SSL_ENABLED_DEFAULT = false;
private static final String HTTP_ADMINS_KEY =

1
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java
View File

@ -28,7 +28,6 @@ public class HttpFSKerberosAuthenticationHandlerForTesting
@Override
public void init(Properties config) throws ServletException {
//NOP overwrite to avoid Kerberos initialization
config.setProperty(TOKEN_KIND, "t");
initTokenManager(config);
}

254
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
View File

@ -20,6 +20,7 @@
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
import org.json.simple.JSONArray;
@ -46,12 +47,14 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttrCodec;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.lib.server.Service;
import org.apache.hadoop.lib.server.ServiceException;
import org.apache.hadoop.lib.service.Groups;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.apache.hadoop.security.authentication.util.Signer;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.HFSTestCase;
import org.apache.hadoop.test.HadoopUsersConfTestHelper;
import org.apache.hadoop.test.TestDir;
@ -120,8 +123,8 @@ public List<String> getGroups(String user) throws IOException {
}
private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
throws Exception {
private Configuration createHttpFSConf(boolean addDelegationTokenAuthHandler,
boolean sslEnabled) throws Exception {
File homeDir = TestDirHelper.getTestDir();
Assert.assertTrue(new File(homeDir, "conf").mkdir());
Assert.assertTrue(new File(homeDir, "log").mkdir());
@ -133,10 +136,11 @@ private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
w.write("secret");
w.close();
//HDFS configuration
// HDFS configuration
File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf");
hadoopConfDir.mkdirs();
Configuration hdfsConf = TestHdfsHelper.getHdfsConf();
// Http Server's conf should be based on HDFS's conf
Configuration conf = new Configuration(hdfsConf);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
@ -146,7 +150,7 @@ private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
conf.writeXml(os);
os.close();
//HTTPFS configuration
// HTTPFS configuration
conf = new Configuration(false);
if (addDelegationTokenAuthHandler) {
conf.set("httpfs.authentication.type",
@ -155,17 +159,30 @@ private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
conf.set("httpfs.services.ext", MockGroups.class.getName());
conf.set("httpfs.admin.group", HadoopUsersConfTestHelper.
getHadoopUserGroups(HadoopUsersConfTestHelper.getHadoopUsers()[0])[0]);
conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups",
conf.set("httpfs.proxyuser." +
HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups",
HadoopUsersConfTestHelper.getHadoopProxyUserGroups());
conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts",
conf.set("httpfs.proxyuser." +
HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts",
HadoopUsersConfTestHelper.getHadoopProxyUserHosts());
conf.set("httpfs.authentication.signature.secret.file", secretFile.getAbsolutePath());
conf.set("httpfs.authentication.signature.secret.file",
secretFile.getAbsolutePath());
conf.set("httpfs.hadoop.config.dir", hadoopConfDir.toString());
if (sslEnabled) {
conf.set("httpfs.ssl.enabled", "true");
}
File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml");
os = new FileOutputStream(httpfsSite);
conf.writeXml(os);
os.close();
return conf;
}
private void createHttpFSServer(boolean addDelegationTokenAuthHandler,
boolean sslEnabled)
throws Exception {
Configuration conf = createHttpFSConf(addDelegationTokenAuthHandler,
sslEnabled);
ClassLoader cl = Thread.currentThread().getContextClassLoader();
URL url = cl.getResource("webapp");
WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs");
@ -177,12 +194,110 @@ private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
}
}
private String getSignedTokenString()
throws Exception {
AuthenticationToken token = new AuthenticationToken("u", "p",
new KerberosDelegationTokenAuthenticationHandler().getType());
token.setExpires(System.currentTimeMillis() + 100000000);
SignerSecretProvider secretProvider =
StringSignerSecretProviderCreator.newStringSignerSecretProvider();
Properties secretProviderProps = new Properties();
secretProviderProps.setProperty(
AuthenticationFilter.SIGNATURE_SECRET, "secret");
secretProvider.init(secretProviderProps, null, -1);
Signer signer = new Signer(secretProvider);
return signer.sign(token.toString());
}
private void delegationTokenCommonTests(boolean sslEnabled) throws Exception {
URL url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETHOMEDIRECTORY");
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
conn.getResponseCode());
String tokenSigned = getSignedTokenString();
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETDELEGATIONTOKEN");
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Cookie",
AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
JSONObject json = (JSONObject)new JSONParser().parse(
new InputStreamReader(conn.getInputStream()));
json = (JSONObject)
json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_JSON);
String tokenStr = (String)json.get(
DelegationTokenAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON);
Token<AbstractDelegationTokenIdentifier> dToken =
new Token<AbstractDelegationTokenIdentifier>();
dToken.decodeFromUrlString(tokenStr);
Assert.assertEquals(sslEnabled ?
WebHdfsConstants.SWEBHDFS_TOKEN_KIND :
WebHdfsConstants.WEBHDFS_TOKEN_KIND,
dToken.getKind());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("PUT");
Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("PUT");
conn.setRequestProperty("Cookie",
AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("PUT");
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
conn.getResponseCode());
// getTrash test with delegation
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETTRASHROOT&delegation=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETTRASHROOT");
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Cookie",
AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
}
@Test
@TestDir
@TestJetty
@TestHdfs
public void instrumentation() throws Exception {
createHttpFSServer(false);
createHttpFSServer(false, false);
URL url = new URL(TestJettyHelper.getJettyURL(),
MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", "nobody"));
@ -211,7 +326,7 @@ public void instrumentation() throws Exception {
@TestJetty
@TestHdfs
public void testHdfsAccess() throws Exception {
createHttpFSServer(false);
createHttpFSServer(false, false);
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
URL url = new URL(TestJettyHelper.getJettyURL(),
@ -228,7 +343,7 @@ public void testHdfsAccess() throws Exception {
@TestJetty
@TestHdfs
public void testMkdirs() throws Exception {
createHttpFSServer(false);
createHttpFSServer(false, false);
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format(
@ -246,7 +361,7 @@ public void testMkdirs() throws Exception {
@TestJetty
@TestHdfs
public void testGlobFilter() throws Exception {
createHttpFSServer(false);
createHttpFSServer(false, false);
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
fs.mkdirs(new Path("/tmp"));
@ -438,7 +553,7 @@ private byte[] decodeXAttrValue(String value) throws IOException {
@TestJetty
@TestHdfs
public void testPerms() throws Exception {
createHttpFSServer(false);
createHttpFSServer(false, false);
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
fs.mkdirs(new Path("/perm"));
@ -475,7 +590,7 @@ public void testXAttrs() throws Exception {
final String dir = "/xattrTest";
final String path = dir + "/file";
createHttpFSServer(false);
createHttpFSServer(false, false);
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
fs.mkdirs(new Path(dir));
@ -544,7 +659,7 @@ public void testFileAcls() throws Exception {
String statusJson;
List<String> aclEntries;
createHttpFSServer(false);
createHttpFSServer(false, false);
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
fs.mkdirs(new Path(dir));
@ -627,7 +742,7 @@ public void testDirAcls() throws Exception {
String statusJson;
List<String> aclEntries;
createHttpFSServer(false);
createHttpFSServer(false, false);
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
fs.mkdirs(new Path(dir));
@ -665,7 +780,7 @@ public void testDirAcls() throws Exception {
@TestJetty
@TestHdfs
public void testOpenOffsetLength() throws Exception {
createHttpFSServer(false);
createHttpFSServer(false, false);
byte[] array = new byte[]{0, 1, 2, 3};
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
@ -690,7 +805,7 @@ public void testOpenOffsetLength() throws Exception {
@TestJetty
@TestHdfs
public void testPutNoOperation() throws Exception {
createHttpFSServer(false);
createHttpFSServer(false, false);
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
URL url = new URL(TestJettyHelper.getJettyURL(),
@ -708,7 +823,7 @@ public void testPutNoOperation() throws Exception {
@TestHdfs
public void testGetTrashRoot() throws Exception {
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
createHttpFSServer(false);
createHttpFSServer(false, false);
String trashJson = getStatus("/", "GETTRASHROOT");
String trashPath = getPath(trashJson);
@ -741,99 +856,16 @@ public void testGetTrashRoot() throws Exception {
@TestJetty
@TestHdfs
public void testDelegationTokenOperations() throws Exception {
createHttpFSServer(true);
URL url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETHOMEDIRECTORY");
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
conn.getResponseCode());
AuthenticationToken token =
new AuthenticationToken("u", "p",
new KerberosDelegationTokenAuthenticationHandler().getType());
token.setExpires(System.currentTimeMillis() + 100000000);
SignerSecretProvider secretProvider =
StringSignerSecretProviderCreator.newStringSignerSecretProvider();
Properties secretProviderProps = new Properties();
secretProviderProps.setProperty(AuthenticationFilter.SIGNATURE_SECRET, "secret");
secretProvider.init(secretProviderProps, null, -1);
Signer signer = new Signer(secretProvider);
String tokenSigned = signer.sign(token.toString());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETHOMEDIRECTORY");
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Cookie",
AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETDELEGATIONTOKEN");
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Cookie",
AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
JSONObject json = (JSONObject)
new JSONParser().parse(new InputStreamReader(conn.getInputStream()));
json = (JSONObject)
json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_JSON);
String tokenStr = (String)
json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON);
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("PUT");
Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("PUT");
conn.setRequestProperty("Cookie",
AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("PUT");
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
conn.getResponseCode());
// getTrash test with delegation
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETTRASHROOT&delegation=" + tokenStr);
conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
conn.getResponseCode());
url = new URL(TestJettyHelper.getJettyURL(),
"/webhdfs/v1/?op=GETTRASHROOT");
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Cookie",
AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
createHttpFSServer(true, false);
delegationTokenCommonTests(false);
}
@Test
@TestDir
@TestJetty
@TestHdfs
public void testDelegationTokenOperationsSsl() throws Exception {
createHttpFSServer(true, true);
delegationTokenCommonTests(true);
}
}