From 201af204061a073fb041b9a7bbe93e35dd43c597 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Fri, 20 Apr 2012 19:32:23 +0000 Subject: [PATCH] Revert r1328482 for HDFS-3308. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1328487 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 - .../apache/hadoop/hdfs/HftpFileSystem.java | 7 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 9 ++- .../hadoop/hdfs/TestHftpDelegationToken.java | 65 ++++------------- .../hadoop/hdfs/web/TestWebHdfsUrl.java | 70 +++++-------------- 5 files changed, 36 insertions(+), 118 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8c29b120d7..f0e4bdd0ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -885,9 +885,6 @@ Release 0.23.3 - UNRELEASED HDFS-2652. Add support for host-based delegation tokens. (Daryn Sharp via szetszwo) - HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem - and WebHdfsFileSystem. (Daryn Sharp via szetszwo) - Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java index cadde41514..7151e9f947 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java @@ -168,7 +168,7 @@ public void initialize(final URI name, final Configuration conf) protected void initDelegationToken() throws IOException { // look for hftp token, then try hdfs - Token token = selectDelegationToken(ugi); + Token token = selectDelegationToken(); // if we don't already have a token, go get one over https boolean createdToken = false; @@ -189,9 +189,8 @@ protected void initDelegationToken() throws IOException { } } - protected Token selectDelegationToken( - UserGroupInformation ugi) { - return hftpTokenSelector.selectToken(getCanonicalUri(), ugi.getTokens(), getConf()); + protected Token selectDelegationToken() { + return hftpTokenSelector.selectToken(getUri(), ugi.getTokens(), getConf()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 4e44e03e05..415bf6c12d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -165,7 +165,7 @@ public synchronized void initialize(URI uri, Configuration conf } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } - this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort()); + this.nnAddr = NetUtils.createSocketAddrForHost(uri.getHost(), uri.getPort()); this.workingDir = getHomeDirectory(); if (UserGroupInformation.isSecurityEnabled()) { @@ -175,7 +175,7 @@ public synchronized void initialize(URI uri, Configuration conf protected void initDelegationToken() throws IOException { // look for webhdfs token, then try hdfs - Token token = selectDelegationToken(ugi); + Token token = selectDelegationToken(); //since we don't already have a token, go get one boolean createdToken = false; @@ -196,9 +196,8 @@ protected void initDelegationToken() throws IOException { } } - protected Token selectDelegationToken( - UserGroupInformation ugi) { - return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf()); + protected Token selectDelegationToken() { + return DT_SELECTOR.selectToken(getUri(), ugi.getTokens(), getConf()); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java index d292aede5c..e407122241 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION; -import java.io.IOException; import java.lang.reflect.Field; import java.net.URI; import java.security.PrivilegedExceptionAction; @@ -74,58 +73,34 @@ public void testSelectHdfsDelegationToken() throws Exception { SecurityUtilTestHelper.setTokenServiceUseIp(true); Configuration conf = new Configuration(); - conf.setClass("fs.hftp.impl", MyHftpFileSystem.class, FileSystem.class); + URI hftpUri = URI.create("hftp://localhost:0"); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + Token token = null; - // test with implicit default port - URI fsUri = URI.create("hftp://localhost"); - MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); - - // test with explicit default port - fsUri = URI.create("hftp://localhost:"+fs.getDefaultPort()); - fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); - - // test with non-default port - fsUri = URI.create("hftp://localhost:"+(fs.getDefaultPort()-1)); - fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); - - } - - private void checkTokenSelection(MyHftpFileSystem fs, - Configuration conf) throws IOException { - int port = fs.getCanonicalUri().getPort(); - UserGroupInformation ugi = - UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{}); - - // use ip-based tokens - SecurityUtilTestHelper.setTokenServiceUseIp(true); - // test fallback to hdfs token Token hdfsToken = new Token( new byte[0], new byte[0], DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text("127.0.0.1:8020")); ugi.addToken(hdfsToken); - - // test fallback to hdfs token - Token token = fs.selectDelegationToken(ugi); + + HftpFileSystem fs = (HftpFileSystem) FileSystem.get(hftpUri, conf); + token = fs.selectDelegationToken(); assertNotNull(token); assertEquals(hdfsToken, token); - + // test hftp is favored over hdfs Token hftpToken = new Token( new byte[0], new byte[0], - HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port)); + HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:0")); ugi.addToken(hftpToken); - token = fs.selectDelegationToken(ugi); + token = fs.selectDelegationToken(); assertNotNull(token); assertEquals(hftpToken, token); // switch to using host-based tokens, no token should match SecurityUtilTestHelper.setTokenServiceUseIp(false); - token = fs.selectDelegationToken(ugi); + token = fs.selectDelegationToken(); assertNull(token); // test fallback to hdfs token @@ -134,31 +109,17 @@ private void checkTokenSelection(MyHftpFileSystem fs, DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text("localhost:8020")); ugi.addToken(hdfsToken); - token = fs.selectDelegationToken(ugi); + token = fs.selectDelegationToken(); assertNotNull(token); assertEquals(hdfsToken, token); // test hftp is favored over hdfs hftpToken = new Token( new byte[0], new byte[0], - HftpFileSystem.TOKEN_KIND, new Text("localhost:"+port)); + HftpFileSystem.TOKEN_KIND, new Text("localhost:0")); ugi.addToken(hftpToken); - token = fs.selectDelegationToken(ugi); + token = fs.selectDelegationToken(); assertNotNull(token); assertEquals(hftpToken, token); } - - static class MyHftpFileSystem extends HftpFileSystem { - @Override - public URI getCanonicalUri() { - return super.getCanonicalUri(); - } - @Override - public int getDefaultPort() { - return super.getDefaultPort(); - } - // don't automatically get a token - @Override - protected void initDelegationToken() throws IOException {} - } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java index 015ce4dde2..1dde0997dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java @@ -98,63 +98,38 @@ private String generateUrlQueryPrefix(HttpOpParam.Op op, String username) { } @Test - public void testSelectHdfsDelegationToken() throws Exception { + public void testSelectDelegationToken() throws Exception { SecurityUtilTestHelper.setTokenServiceUseIp(true); Configuration conf = new Configuration(); - conf.setClass("fs.webhdfs.impl", MyWebHdfsFileSystem.class, FileSystem.class); + URI webHdfsUri = URI.create("webhdfs://localhost:0"); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + Token token = null; - // test with implicit default port - URI fsUri = URI.create("webhdfs://localhost"); - MyWebHdfsFileSystem fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); - - // test with explicit default port - fsUri = URI.create("webhdfs://localhost:"+fs.getDefaultPort()); - fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); - - // test with non-default port - fsUri = URI.create("webhdfs://localhost:"+(fs.getDefaultPort()-1)); - fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); - - } - - private void checkTokenSelection(MyWebHdfsFileSystem fs, - Configuration conf) throws IOException { - int port = fs.getCanonicalUri().getPort(); - // can't clear tokens from ugi, so create a new user everytime - UserGroupInformation ugi = - UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{}); - - // use ip-based tokens - SecurityUtilTestHelper.setTokenServiceUseIp(true); - // test fallback to hdfs token Token hdfsToken = new Token( new byte[0], new byte[0], DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text("127.0.0.1:8020")); ugi.addToken(hdfsToken); - - // test fallback to hdfs token - Token token = fs.selectDelegationToken(ugi); + + WebHdfsFileSystem fs = (WebHdfsFileSystem) FileSystem.get(webHdfsUri, conf); + token = fs.selectDelegationToken(); assertNotNull(token); assertEquals(hdfsToken, token); - + // test webhdfs is favored over hdfs Token webHdfsToken = new Token( new byte[0], new byte[0], - WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port)); + WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:0")); ugi.addToken(webHdfsToken); - token = fs.selectDelegationToken(ugi); + token = fs.selectDelegationToken(); assertNotNull(token); assertEquals(webHdfsToken, token); // switch to using host-based tokens, no token should match SecurityUtilTestHelper.setTokenServiceUseIp(false); - token = fs.selectDelegationToken(ugi); + token = fs.selectDelegationToken(); assertNull(token); // test fallback to hdfs token @@ -163,31 +138,18 @@ private void checkTokenSelection(MyWebHdfsFileSystem fs, DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text("localhost:8020")); ugi.addToken(hdfsToken); - token = fs.selectDelegationToken(ugi); + token = fs.selectDelegationToken(); assertNotNull(token); assertEquals(hdfsToken, token); // test webhdfs is favored over hdfs webHdfsToken = new Token( new byte[0], new byte[0], - WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:"+port)); + WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:0")); ugi.addToken(webHdfsToken); - token = fs.selectDelegationToken(ugi); + token = fs.selectDelegationToken(); assertNotNull(token); assertEquals(webHdfsToken, token); } - - static class MyWebHdfsFileSystem extends WebHdfsFileSystem { - @Override - public URI getCanonicalUri() { - return super.getCanonicalUri(); - } - @Override - public int getDefaultPort() { - return super.getDefaultPort(); - } - // don't automatically get a token - @Override - protected void initDelegationToken() throws IOException {} - } -} \ No newline at end of file + +}