big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert r1328482 for HDFS-3308.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1328487 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2012-04-20 19:32:23 +00:00
parent ab8f458742
commit 201af20406
5 changed files with 36 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -885,9 +885,6 @@ Release 0.23.3 - UNRELEASED
HDFS-2652. Add support for host-based delegation tokens. (Daryn Sharp via HDFS-2652. Add support for host-based delegation tokens. (Daryn Sharp via
szetszwo) szetszwo)
HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem
and WebHdfsFileSystem. (Daryn Sharp via szetszwo)
Release 0.23.2 - UNRELEASED Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
View File

@ -168,7 +168,7 @@ public void initialize(final URI name, final Configuration conf)
protected void initDelegationToken() throws IOException { protected void initDelegationToken() throws IOException {
// look for hftp token, then try hdfs // look for hftp token, then try hdfs
Token<?> token = selectDelegationToken(ugi); Token<?> token = selectDelegationToken();
// if we don't already have a token, go get one over https // if we don't already have a token, go get one over https
boolean createdToken = false; boolean createdToken = false;
@ -189,9 +189,8 @@ protected void initDelegationToken() throws IOException {
} }
} }
protected Token<DelegationTokenIdentifier> selectDelegationToken( protected Token<DelegationTokenIdentifier> selectDelegationToken() {
UserGroupInformation ugi) { return hftpTokenSelector.selectToken(getUri(), ugi.getTokens(), getConf());
return hftpTokenSelector.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
} }

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
View File

@ -165,7 +165,7 @@ public synchronized void initialize(URI uri, Configuration conf
} catch (URISyntaxException e) { } catch (URISyntaxException e) {
throw new IllegalArgumentException(e); throw new IllegalArgumentException(e);
} }
this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort()); this.nnAddr = NetUtils.createSocketAddrForHost(uri.getHost(), uri.getPort());
this.workingDir = getHomeDirectory(); this.workingDir = getHomeDirectory();
if (UserGroupInformation.isSecurityEnabled()) { if (UserGroupInformation.isSecurityEnabled()) {
@ -175,7 +175,7 @@ public synchronized void initialize(URI uri, Configuration conf
protected void initDelegationToken() throws IOException { protected void initDelegationToken() throws IOException {
// look for webhdfs token, then try hdfs // look for webhdfs token, then try hdfs
Token<?> token = selectDelegationToken(ugi); Token<?> token = selectDelegationToken();
//since we don't already have a token, go get one //since we don't already have a token, go get one
boolean createdToken = false; boolean createdToken = false;
@ -196,9 +196,8 @@ protected void initDelegationToken() throws IOException {
} }
} }
protected Token<DelegationTokenIdentifier> selectDelegationToken( protected Token<DelegationTokenIdentifier> selectDelegationToken() {
UserGroupInformation ugi) { return DT_SELECTOR.selectToken(getUri(), ugi.getTokens(), getConf());
return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
} }
@Override @Override

65
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
View File

@ -21,7 +21,6 @@
import static import static
org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION; org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
import java.io.IOException;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.net.URI; import java.net.URI;
import java.security.PrivilegedExceptionAction; import java.security.PrivilegedExceptionAction;
@ -74,58 +73,34 @@ public void testSelectHdfsDelegationToken() throws Exception {
SecurityUtilTestHelper.setTokenServiceUseIp(true); SecurityUtilTestHelper.setTokenServiceUseIp(true);
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.setClass("fs.hftp.impl", MyHftpFileSystem.class, FileSystem.class); URI hftpUri = URI.create("hftp://localhost:0");
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
Token<?> token = null;
// test with implicit default port
URI fsUri = URI.create("hftp://localhost");
MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
checkTokenSelection(fs, conf);
// test with explicit default port
fsUri = URI.create("hftp://localhost:"+fs.getDefaultPort());
fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
checkTokenSelection(fs, conf);
// test with non-default port
fsUri = URI.create("hftp://localhost:"+(fs.getDefaultPort()-1));
fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
checkTokenSelection(fs, conf);
}
private void checkTokenSelection(MyHftpFileSystem fs,
Configuration conf) throws IOException {
int port = fs.getCanonicalUri().getPort();
UserGroupInformation ugi =
UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{});
// use ip-based tokens
SecurityUtilTestHelper.setTokenServiceUseIp(true);
// test fallback to hdfs token // test fallback to hdfs token
Token<?> hdfsToken = new Token<TokenIdentifier>( Token<?> hdfsToken = new Token<TokenIdentifier>(
new byte[0], new byte[0], new byte[0], new byte[0],
DelegationTokenIdentifier.HDFS_DELEGATION_KIND, DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
new Text("127.0.0.1:8020")); new Text("127.0.0.1:8020"));
ugi.addToken(hdfsToken); ugi.addToken(hdfsToken);
// test fallback to hdfs token HftpFileSystem fs = (HftpFileSystem) FileSystem.get(hftpUri, conf);
Token<?> token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNotNull(token); assertNotNull(token);
assertEquals(hdfsToken, token); assertEquals(hdfsToken, token);
// test hftp is favored over hdfs // test hftp is favored over hdfs
Token<?> hftpToken = new Token<TokenIdentifier>( Token<?> hftpToken = new Token<TokenIdentifier>(
new byte[0], new byte[0], new byte[0], new byte[0],
HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port)); HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:0"));
ugi.addToken(hftpToken); ugi.addToken(hftpToken);
token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNotNull(token); assertNotNull(token);
assertEquals(hftpToken, token); assertEquals(hftpToken, token);
// switch to using host-based tokens, no token should match // switch to using host-based tokens, no token should match
SecurityUtilTestHelper.setTokenServiceUseIp(false); SecurityUtilTestHelper.setTokenServiceUseIp(false);
token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNull(token); assertNull(token);
// test fallback to hdfs token // test fallback to hdfs token
@ -134,31 +109,17 @@ private void checkTokenSelection(MyHftpFileSystem fs,
DelegationTokenIdentifier.HDFS_DELEGATION_KIND, DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
new Text("localhost:8020")); new Text("localhost:8020"));
ugi.addToken(hdfsToken); ugi.addToken(hdfsToken);
token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNotNull(token); assertNotNull(token);
assertEquals(hdfsToken, token); assertEquals(hdfsToken, token);
// test hftp is favored over hdfs // test hftp is favored over hdfs
hftpToken = new Token<TokenIdentifier>( hftpToken = new Token<TokenIdentifier>(
new byte[0], new byte[0], new byte[0], new byte[0],
HftpFileSystem.TOKEN_KIND, new Text("localhost:"+port)); HftpFileSystem.TOKEN_KIND, new Text("localhost:0"));
ugi.addToken(hftpToken); ugi.addToken(hftpToken);
token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNotNull(token); assertNotNull(token);
assertEquals(hftpToken, token); assertEquals(hftpToken, token);
} }
static class MyHftpFileSystem extends HftpFileSystem {
@Override
public URI getCanonicalUri() {
return super.getCanonicalUri();
}
@Override
public int getDefaultPort() {
return super.getDefaultPort();
}
// don't automatically get a token
@Override
protected void initDelegationToken() throws IOException {}
}
} }

70
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
View File

@ -98,63 +98,38 @@ private String generateUrlQueryPrefix(HttpOpParam.Op op, String username) {
} }
@Test @Test
public void testSelectHdfsDelegationToken() throws Exception { public void testSelectDelegationToken() throws Exception {
SecurityUtilTestHelper.setTokenServiceUseIp(true); SecurityUtilTestHelper.setTokenServiceUseIp(true);
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.setClass("fs.webhdfs.impl", MyWebHdfsFileSystem.class, FileSystem.class); URI webHdfsUri = URI.create("webhdfs://localhost:0");
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
Token<?> token = null;
// test with implicit default port
URI fsUri = URI.create("webhdfs://localhost");
MyWebHdfsFileSystem fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf);
checkTokenSelection(fs, conf);
// test with explicit default port
fsUri = URI.create("webhdfs://localhost:"+fs.getDefaultPort());
fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf);
checkTokenSelection(fs, conf);
// test with non-default port
fsUri = URI.create("webhdfs://localhost:"+(fs.getDefaultPort()-1));
fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf);
checkTokenSelection(fs, conf);
}
private void checkTokenSelection(MyWebHdfsFileSystem fs,
Configuration conf) throws IOException {
int port = fs.getCanonicalUri().getPort();
// can't clear tokens from ugi, so create a new user everytime
UserGroupInformation ugi =
UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{});
// use ip-based tokens
SecurityUtilTestHelper.setTokenServiceUseIp(true);
// test fallback to hdfs token // test fallback to hdfs token
Token<?> hdfsToken = new Token<TokenIdentifier>( Token<?> hdfsToken = new Token<TokenIdentifier>(
new byte[0], new byte[0], new byte[0], new byte[0],
DelegationTokenIdentifier.HDFS_DELEGATION_KIND, DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
new Text("127.0.0.1:8020")); new Text("127.0.0.1:8020"));
ugi.addToken(hdfsToken); ugi.addToken(hdfsToken);
// test fallback to hdfs token WebHdfsFileSystem fs = (WebHdfsFileSystem) FileSystem.get(webHdfsUri, conf);
Token<?> token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNotNull(token); assertNotNull(token);
assertEquals(hdfsToken, token); assertEquals(hdfsToken, token);
// test webhdfs is favored over hdfs // test webhdfs is favored over hdfs
Token<?> webHdfsToken = new Token<TokenIdentifier>( Token<?> webHdfsToken = new Token<TokenIdentifier>(
new byte[0], new byte[0], new byte[0], new byte[0],
WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port)); WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:0"));
ugi.addToken(webHdfsToken); ugi.addToken(webHdfsToken);
token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNotNull(token); assertNotNull(token);
assertEquals(webHdfsToken, token); assertEquals(webHdfsToken, token);
// switch to using host-based tokens, no token should match // switch to using host-based tokens, no token should match
SecurityUtilTestHelper.setTokenServiceUseIp(false); SecurityUtilTestHelper.setTokenServiceUseIp(false);
token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNull(token); assertNull(token);
// test fallback to hdfs token // test fallback to hdfs token
@ -163,31 +138,18 @@ private void checkTokenSelection(MyWebHdfsFileSystem fs,
DelegationTokenIdentifier.HDFS_DELEGATION_KIND, DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
new Text("localhost:8020")); new Text("localhost:8020"));
ugi.addToken(hdfsToken); ugi.addToken(hdfsToken);
token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNotNull(token); assertNotNull(token);
assertEquals(hdfsToken, token); assertEquals(hdfsToken, token);
// test webhdfs is favored over hdfs // test webhdfs is favored over hdfs
webHdfsToken = new Token<TokenIdentifier>( webHdfsToken = new Token<TokenIdentifier>(
new byte[0], new byte[0], new byte[0], new byte[0],
WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:"+port)); WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:0"));
ugi.addToken(webHdfsToken); ugi.addToken(webHdfsToken);
token = fs.selectDelegationToken(ugi); token = fs.selectDelegationToken();
assertNotNull(token); assertNotNull(token);
assertEquals(webHdfsToken, token); assertEquals(webHdfsToken, token);
} }
static class MyWebHdfsFileSystem extends WebHdfsFileSystem { }
@Override
public URI getCanonicalUri() {
return super.getCanonicalUri();
}
@Override
public int getDefaultPort() {
return super.getDefaultPort();
}
// don't automatically get a token
@Override
protected void initDelegationToken() throws IOException {}
}
}