big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-3312. In HftpFileSystem, the namenode URI is non-secure but the delegation tokens have to use secure URI. Contributed by Daryn Sharp

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1329462 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2012-04-23 21:27:01 +00:00
parent 80ab78c217
commit 93c5585d31
4 changed files with 94 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -890,6 +890,9 @@ Release 0.23.3 - UNRELEASED
HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem
and WebHdfsFileSystem. (Daryn Sharp via szetszwo)
HDFS-3312. In HftpFileSystem, the namenode URI is non-secure but the
delegation tokens have to use secure URI. (Daryn Sharp via szetszwo)
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
View File

@ -94,8 +94,8 @@ public class HftpFileSystem extends FileSystem
protected UserGroupInformation ugi;
private URI hftpURI;
protected InetSocketAddress nnAddr;
protected InetSocketAddress nnSecureAddr;
protected URI nnUri;
protected URI nnSecureUri;
public static final String HFTP_TIMEZONE = "UTC";
public static final String HFTP_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
@ -139,11 +139,19 @@ protected InetSocketAddress getNamenodeSecureAddr(URI uri) {
return NetUtils.createSocketAddrForHost(uri.getHost(), getDefaultSecurePort());
}
protected URI getNamenodeUri(URI uri) {
return DFSUtil.createUri("http", getNamenodeAddr(uri));
}
protected URI getNamenodeSecureUri(URI uri) {
return DFSUtil.createUri("https", getNamenodeSecureAddr(uri));
}
@Override
public String getCanonicalServiceName() {
// unlike other filesystems, hftp's service is the secure port, not the
// actual port in the uri
return SecurityUtil.buildTokenService(nnSecureAddr).toString();
return SecurityUtil.buildTokenService(nnSecureUri).toString();
}
@Override
@ -152,8 +160,8 @@ public void initialize(final URI name, final Configuration conf)
super.initialize(name, conf);
setConf(conf);
this.ugi = UserGroupInformation.getCurrentUser();
this.nnAddr = getNamenodeAddr(name);
this.nnSecureAddr = getNamenodeSecureAddr(name);
this.nnUri = getNamenodeUri(name);
this.nnSecureUri = getNamenodeSecureUri(name);
try {
this.hftpURI = new URI(name.getScheme(), name.getAuthority(),
null, null, null);
@ -191,7 +199,7 @@ protected void initDelegationToken() throws IOException {
protected Token<DelegationTokenIdentifier> selectDelegationToken(
UserGroupInformation ugi) {
return hftpTokenSelector.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
return hftpTokenSelector.selectToken(nnSecureUri, ugi.getTokens(), getConf());
}
@ -222,7 +230,7 @@ public synchronized Token<?> getDelegationToken(final String renewer
ugi.reloginFromKeytab();
return ugi.doAs(new PrivilegedExceptionAction<Token<?>>() {
public Token<?> run() throws IOException {
final String nnHttpUrl = DFSUtil.createUri("https", nnSecureAddr).toString();
final String nnHttpUrl = nnSecureUri.toString();
Credentials c;
try {
c = DelegationTokenFetcher.getDTfromRemote(nnHttpUrl, renewer);
@ -264,8 +272,8 @@ public URI getUri() {
* @throws IOException on error constructing the URL
*/
protected URL getNamenodeURL(String path, String query) throws IOException {
final URL url = new URL("http", nnAddr.getHostName(),
nnAddr.getPort(), path + '?' + query);
final URL url = new URL("http", nnUri.getHost(),
nnUri.getPort(), path + '?' + query);
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
}

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
View File

@ -132,12 +132,17 @@ protected InetSocketAddress getNamenodeSecureAddr(URI uri) {
return getNamenodeAddr(uri);
}
@Override
protected URI getNamenodeUri(URI uri) {
return getNamenodeSecureUri(uri);
}
@Override
protected HttpURLConnection openConnection(String path, String query)
throws IOException {
query = addDelegationTokenParam(query);
final URL url = new URL("https", nnAddr.getHostName(),
nnAddr.getPort(), path + '?' + query);
final URL url = new URL("https", nnUri.getHost(),
nnUri.getPort(), path + '?' + query);
HttpsURLConnection conn = (HttpsURLConnection)URLUtils.openConnection(url);
// bypass hostname verification
conn.setHostnameVerifier(new DummyHostnameVerifier());

78
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
View File

@ -70,32 +70,74 @@ public FileSystem run() throws Exception {
}
@Test
public void testSelectHdfsDelegationToken() throws Exception {
public void testSelectHftpDelegationToken() throws Exception {
SecurityUtilTestHelper.setTokenServiceUseIp(true);
Configuration conf = new Configuration();
conf.setClass("fs.hftp.impl", MyHftpFileSystem.class, FileSystem.class);
int httpPort = 80;
int httpsPort = 443;
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, httpPort);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, httpsPort);
// test with implicit default port
URI fsUri = URI.create("hftp://localhost");
MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
checkTokenSelection(fs, conf);
MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.newInstance(fsUri, conf);
assertEquals(httpPort, fs.getCanonicalUri().getPort());
checkTokenSelection(fs, httpsPort, conf); // should still use secure port
// test with explicit default port
fsUri = URI.create("hftp://localhost:"+fs.getDefaultPort());
fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
checkTokenSelection(fs, conf);
fsUri = URI.create("hftp://localhost:"+httpPort);
fs = (MyHftpFileSystem) FileSystem.newInstance(fsUri, conf);
assertEquals(httpPort, fs.getCanonicalUri().getPort());
checkTokenSelection(fs, httpsPort, conf); // should still use secure port
// test with non-default port
fsUri = URI.create("hftp://localhost:"+(fs.getDefaultPort()-1));
fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
checkTokenSelection(fs, conf);
fsUri = URI.create("hftp://localhost:"+(httpPort+1));
fs = (MyHftpFileSystem) FileSystem.newInstance(fsUri, conf);
assertEquals(httpPort+1, fs.getCanonicalUri().getPort());
checkTokenSelection(fs, httpsPort, conf); // should still use secure port
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 5);
}
private void checkTokenSelection(MyHftpFileSystem fs,
@Test
public void testSelectHsftpDelegationToken() throws Exception {
SecurityUtilTestHelper.setTokenServiceUseIp(true);
Configuration conf = new Configuration();
conf.setClass("fs.hsftp.impl", MyHsftpFileSystem.class, FileSystem.class);
int httpPort = 80;
int httpsPort = 443;
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, httpPort);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, httpsPort);
// test with implicit default port
URI fsUri = URI.create("hsftp://localhost");
MyHsftpFileSystem fs = (MyHsftpFileSystem) FileSystem.newInstance(fsUri, conf);
assertEquals(httpsPort, fs.getCanonicalUri().getPort());
checkTokenSelection(fs, httpsPort, conf);
// test with explicit default port
fsUri = URI.create("hsftp://localhost:"+httpsPort);
fs = (MyHsftpFileSystem) FileSystem.newInstance(fsUri, conf);
assertEquals(httpsPort, fs.getCanonicalUri().getPort());
checkTokenSelection(fs, httpsPort, conf);
// test with non-default port
fsUri = URI.create("hsftp://localhost:"+(httpsPort+1));
fs = (MyHsftpFileSystem) FileSystem.newInstance(fsUri, conf);
assertEquals(httpsPort+1, fs.getCanonicalUri().getPort());
checkTokenSelection(fs, httpsPort+1, conf);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 5);
}
private void checkTokenSelection(HftpFileSystem fs,
int port,
Configuration conf) throws IOException {
int port = fs.getCanonicalUri().getPort();
UserGroupInformation ugi =
UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{});
@ -161,4 +203,18 @@ public int getDefaultPort() {
@Override
protected void initDelegationToken() throws IOException {}
}
static class MyHsftpFileSystem extends HsftpFileSystem {
@Override
public URI getCanonicalUri() {
return super.getCanonicalUri();
}
@Override
public int getDefaultPort() {
return super.getDefaultPort();
}
// don't automatically get a token
@Override
protected void initDelegationToken() throws IOException {}
}
}