From 620890fcc0fb8680a3ad282b0b7b969277deb766 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 15 Nov 2013 02:11:38 +0000 Subject: [PATCH] HDFS-5489. Use TokenAspect in WebHDFSFileSystem. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1542158 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../apache/hadoop/hdfs/web/TokenAspect.java | 4 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 75 ++----- .../hadoop/hdfs/web/TestTokenAspect.java | 89 +++++++- .../hadoop/hdfs/web/TestWebHdfsTokens.java | 191 ++++-------------- .../hadoop/hdfs/web/TestWebHdfsUrl.java | 106 +--------- 6 files changed, 157 insertions(+), 310 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2eef17a505..32e1df676c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -494,6 +494,8 @@ Release 2.3.0 - UNRELEASED HDFS-5506. Use URLConnectionFactory in DelegationTokenFetcher. (Haohui Mai via jing9) + HDFS-5489. Use TokenAspect in WebHDFSFileSystem. (Haohui Mai via jing9) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java index 5464ba0c75..f3fac077f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java @@ -144,6 +144,10 @@ synchronized void ensureTokenInitialized() throws IOException { } } + public synchronized void reset() { + hasInitedToken = false; + } + synchronized void initDelegationToken(UserGroupInformation ugi) { Token token = selectDelegationToken(ugi); if (token != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index f293e71e74..aa631a23e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -118,38 +118,11 @@ public class WebHdfsFileSystem extends FileSystem /** Delegation token kind */ public static final Text TOKEN_KIND = new Text("WEBHDFS delegation"); - /** Token selector */ - public static final DTSelecorByKind DT_SELECTOR - = new DTSelecorByKind(TOKEN_KIND); - - private DelegationTokenRenewer dtRenewer = null; - @VisibleForTesting - DelegationTokenRenewer.RenewAction action; - - @Override - public URI getCanonicalUri() { - return super.getCanonicalUri(); - } - - @VisibleForTesting - protected synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) { - if (dtRenewer == null) { - dtRenewer = DelegationTokenRenewer.getInstance(); - } - - action = dtRenewer.addRenewAction(webhdfs); - } - - /** Is WebHDFS enabled in conf? */ - public static boolean isEnabled(final Configuration conf, final Log log) { - final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, - DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT); - return b; - } + protected TokenAspect tokenAspect = new TokenAspect( + this, TOKEN_KIND); private UserGroupInformation ugi; private URI uri; - private boolean hasInitedToken; private Token delegationToken; private RetryPolicy retryPolicy = null; private Path workingDir; @@ -212,41 +185,27 @@ public synchronized void initialize(URI uri, Configuration conf this.workingDir = getHomeDirectory(); if (UserGroupInformation.isSecurityEnabled()) { - initDelegationToken(); + tokenAspect.initDelegationToken(ugi); } } - protected void initDelegationToken() throws IOException { - // look for webhdfs token, then try hdfs - Token token = selectDelegationToken(ugi); - if (token != null) { - LOG.debug("Found existing DT for " + token.getService()); - setDelegationToken(token); - hasInitedToken = true; - } + @Override + public URI getCanonicalUri() { + return super.getCanonicalUri(); + } + + /** Is WebHDFS enabled in conf? */ + public static boolean isEnabled(final Configuration conf, final Log log) { + final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, + DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT); + return b; } protected synchronized Token getDelegationToken() throws IOException { - // we haven't inited yet, or we used to have a token but it expired - if (!hasInitedToken || (action != null && !action.isValid())) { - //since we don't already have a token, go get one - Token token = getDelegationToken(null); - // security might be disabled - if (token != null) { - setDelegationToken(token); - addRenewAction(this); - LOG.debug("Created new DT for " + token.getService()); - } - hasInitedToken = true; - } + tokenAspect.ensureTokenInitialized(); return delegationToken; } - protected Token selectDelegationToken( - UserGroupInformation ugi) { - return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf()); - } - @Override protected int getDefaultPort() { return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, @@ -370,7 +329,7 @@ private synchronized InetSocketAddress getCurrentNNAddr() { private synchronized void resetStateToFailOver() { currentNNAddrIndex = (currentNNAddrIndex + 1) % nnAddrs.length; delegationToken = null; - hasInitedToken = false; + tokenAspect.reset(); } /** @@ -881,9 +840,7 @@ public FSDataInputStream open(final Path f, final int buffersize @Override public void close() throws IOException { super.close(); - if (dtRenewer != null) { - dtRenewer.removeRenewAction(this); // blocks - } + tokenAspect.removeRenewAction(); } class OffsetUrlOpener extends ByteRangeInputStream.URLOpener { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java index abee3a2556..ce5e7a8c43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java @@ -19,13 +19,19 @@ package org.apache.hadoop.hdfs.web; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import java.io.FileNotFoundException; @@ -35,6 +41,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.DelegationTokenRenewer; +import org.apache.hadoop.fs.DelegationTokenRenewer.RenewAction; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -163,15 +170,44 @@ public void setWorkingDirectory(Path new_dir) { } } + private static RenewAction getActionFromTokenAspect( + TokenAspect tokenAspect) { + return (RenewAction) Whitebox.getInternalState(tokenAspect, "action"); + } + @Test - public void testGetRemoteToken() throws IOException, URISyntaxException { + public void testCachedInitialization() throws IOException, URISyntaxException { Configuration conf = new Configuration(); - UserGroupInformation.setConfiguration(conf); DummyFs fs = spy(new DummyFs()); Token token = new Token(new byte[0], new byte[0], DummyFs.TOKEN_KIND, new Text("127.0.0.1:1234")); doReturn(token).when(fs).getDelegationToken(anyString()); + doReturn(token).when(fs).getRenewToken(); + + fs.emulateSecurityEnabled = true; + fs.initialize(new URI("dummyfs://127.0.0.1:1234"), conf); + + fs.tokenAspect.ensureTokenInitialized(); + verify(fs, times(1)).getDelegationToken(null); + verify(fs, times(1)).setDelegationToken(token); + + // For the second iteration, the token should be cached. + fs.tokenAspect.ensureTokenInitialized(); + verify(fs, times(1)).getDelegationToken(null); + verify(fs, times(1)).setDelegationToken(token); + } + + @Test + public void testGetRemoteToken() throws IOException, URISyntaxException { + Configuration conf = new Configuration(); + DummyFs fs = spy(new DummyFs()); + Token token = new Token(new byte[0], + new byte[0], DummyFs.TOKEN_KIND, new Text("127.0.0.1:1234")); + + doReturn(token).when(fs).getDelegationToken(anyString()); + doReturn(token).when(fs).getRenewToken(); + fs.initialize(new URI("dummyfs://127.0.0.1:1234"), conf); fs.tokenAspect.ensureTokenInitialized(); @@ -186,7 +222,6 @@ public void testGetRemoteToken() throws IOException, URISyntaxException { public void testGetRemoteTokenFailure() throws IOException, URISyntaxException { Configuration conf = new Configuration(); - UserGroupInformation.setConfiguration(conf); DummyFs fs = spy(new DummyFs()); IOException e = new IOException(); doThrow(e).when(fs).getDelegationToken(anyString()); @@ -203,7 +238,6 @@ public void testGetRemoteTokenFailure() throws IOException, @Test public void testInitWithNoTokens() throws IOException, URISyntaxException { Configuration conf = new Configuration(); - UserGroupInformation.setConfiguration(conf); DummyFs fs = spy(new DummyFs()); doReturn(null).when(fs).getDelegationToken(anyString()); fs.initialize(new URI("dummyfs://127.0.0.1:1234"), conf); @@ -218,7 +252,6 @@ public void testInitWithNoTokens() throws IOException, URISyntaxException { @Test public void testInitWithUGIToken() throws IOException, URISyntaxException { Configuration conf = new Configuration(); - UserGroupInformation.setConfiguration(conf); DummyFs fs = spy(new DummyFs()); doReturn(null).when(fs).getDelegationToken(anyString()); @@ -241,6 +274,51 @@ public void testInitWithUGIToken() throws IOException, URISyntaxException { assertNull(Whitebox.getInternalState(fs.tokenAspect, "action")); } + @Test + public void testRenewal() throws Exception { + Configuration conf = new Configuration(); + Token token1 = mock(Token.class); + Token token2 = mock(Token.class); + final long renewCycle = 100; + DelegationTokenRenewer.renewCycle = renewCycle; + + UserGroupInformation ugi = UserGroupInformation.createUserForTesting("foo", + new String[] { "bar" }); + DummyFs fs = spy(new DummyFs()); + + doReturn(token1).doReturn(token2).when(fs).getDelegationToken(null); + doReturn(token1).when(fs).getRenewToken(); + // cause token renewer to abandon the token + doThrow(new IOException("renew failed")).when(token1).renew(conf); + doThrow(new IOException("get failed")).when(fs).addDelegationTokens(null, + null); + + TokenAspect tokenAspect = new TokenAspect(fs, + DummyFs.TOKEN_KIND); + fs.initialize(new URI("dummyfs://127.0.0.1:1234"), conf); + tokenAspect.initDelegationToken(ugi); + + // trigger token acquisition + tokenAspect.ensureTokenInitialized(); + DelegationTokenRenewer.RenewAction action = getActionFromTokenAspect(tokenAspect); + verify(fs).setDelegationToken(token1); + assertTrue(action.isValid()); + + // upon renewal, token will go bad based on above stubbing + Thread.sleep(renewCycle * 2); + assertSame(action, getActionFromTokenAspect(tokenAspect)); + assertFalse(action.isValid()); + + // now that token is invalid, should get a new one + tokenAspect.ensureTokenInitialized(); + verify(fs, times(2)).getDelegationToken(anyString()); + verify(fs).setDelegationToken(token2); + assertNotSame(action, getActionFromTokenAspect(tokenAspect)); + + action = getActionFromTokenAspect(tokenAspect); + assertTrue(action.isValid()); + } + @Test public void testTokenSelectionPreferences() throws IOException, URISyntaxException { @@ -252,7 +330,6 @@ public void testTokenSelectionPreferences() throws IOException, DummyFs.TOKEN_KIND); UserGroupInformation ugi = UserGroupInformation.createUserForTesting("foo", new String[] { "bar" }); - UserGroupInformation.setConfiguration(conf); // use ip-based tokens SecurityUtilTestHelper.setTokenServiceUseIp(true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java index 38d5d78788..e649da3c28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java @@ -19,16 +19,20 @@ package org.apache.hadoop.hdfs.web; import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; -import static org.junit.Assert.*; -import static org.mockito.Matchers.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import java.io.IOException; import java.net.URI; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.DelegationTokenRenewer; -import org.apache.hadoop.fs.DelegationTokenRenewer.RenewAction; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.web.resources.DeleteOpParam; import org.apache.hadoop.hdfs.web.resources.GetOpParam; @@ -40,211 +44,102 @@ import org.apache.hadoop.security.token.Token; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; public class TestWebHdfsTokens { - static Configuration conf; - static UserGroupInformation ugi; - + private static Configuration conf; + @BeforeClass - public static void setup() throws IOException { + public static void setUp() { conf = new Configuration(); SecurityUtil.setAuthenticationMethod(KERBEROS, conf); UserGroupInformation.setConfiguration(conf); - ugi = UserGroupInformation.getCurrentUser(); } - - @SuppressWarnings("unchecked") - @Test(timeout=1000) - public void testInitWithNoToken() throws IOException { - WebHdfsFileSystem fs = spy(new WebHdfsFileSystem()); - doReturn(null).when(fs).getDelegationToken(anyString()); - doNothing().when(fs).addRenewAction(any(WebHdfsFileSystem.class)); - fs.initialize(URI.create("webhdfs://127.0.0.1:0"), conf); - - // when not in ugi, don't get one - verify(fs).initDelegationToken(); - verify(fs).selectDelegationToken(ugi); - verify(fs, never()).setDelegationToken(any(Token.class)); - verify(fs, never()).getDelegationToken(); - verify(fs, never()).getDelegationToken(anyString()); + + private WebHdfsFileSystem spyWebhdfsInSecureSetup() throws IOException { + WebHdfsFileSystem fsOrig = new WebHdfsFileSystem(); + fsOrig.initialize(URI.create("webhdfs://127.0.0.1:0"), conf); + WebHdfsFileSystem fs = spy(fsOrig); + Whitebox.setInternalState(fsOrig.tokenAspect, "fs", fs); + return fs; } @SuppressWarnings("unchecked") - @Test(timeout=1000) - public void testInitWithUGIToken() throws IOException { - WebHdfsFileSystem fs = spy(new WebHdfsFileSystem()); - Token token = mock(Token.class); - doReturn(token).when(fs).selectDelegationToken(ugi); - doReturn(null).when(fs).getDelegationToken(anyString()); - doNothing().when(fs).addRenewAction(any(WebHdfsFileSystem.class)); - fs.initialize(URI.create("webhdfs://127.0.0.1:0"), conf); - - // when in the ugi, store it but don't renew it - verify(fs).initDelegationToken(); - verify(fs).selectDelegationToken(ugi); - verify(fs).setDelegationToken(token); - verify(fs, never()).getDelegationToken(); - verify(fs, never()).getDelegationToken(anyString()); - verify(fs, never()).addRenewAction(fs); - } - - @SuppressWarnings("unchecked") - @Test(timeout=1000) - public void testInternalGetDelegationToken() throws IOException { - WebHdfsFileSystem fs = spy(new WebHdfsFileSystem()); - Token token = mock(Token.class); - doReturn(null).when(fs).selectDelegationToken(ugi); - doReturn(token).when(fs).getDelegationToken(anyString()); - doNothing().when(fs).addRenewAction(any(WebHdfsFileSystem.class)); - fs.initialize(URI.create("webhdfs://127.0.0.1:0"), conf); - - // get token, store it, and renew it - Token token2 = fs.getDelegationToken(); - assertEquals(token2, token); - verify(fs).getDelegationToken(null); - verify(fs).setDelegationToken(token); - verify(fs).addRenewAction(fs); - reset(fs); - - // just return token, don't get/set/renew - token2 = fs.getDelegationToken(); - assertEquals(token2, token); - verify(fs, never()).getDelegationToken(null); - verify(fs, never()).setDelegationToken(any(Token.class)); - verify(fs, never()).addRenewAction(fs); - } - - @SuppressWarnings("unchecked") - @Test(timeout=1000) + @Test(timeout = 1000) public void testTokenForNonTokenOp() throws IOException { - WebHdfsFileSystem fs = spy(new WebHdfsFileSystem()); - Token token = mock(Token.class); - doReturn(null).when(fs).selectDelegationToken(ugi); + WebHdfsFileSystem fs = spyWebhdfsInSecureSetup(); + Token token = mock(Token.class); doReturn(token).when(fs).getDelegationToken(null); - doNothing().when(fs).addRenewAction(any(WebHdfsFileSystem.class)); - fs.initialize(URI.create("webhdfs://127.0.0.1:0"), conf); // should get/set/renew token fs.toUrl(GetOpParam.Op.OPEN, null); verify(fs).getDelegationToken(); verify(fs).getDelegationToken(null); verify(fs).setDelegationToken(token); - verify(fs).addRenewAction(fs); reset(fs); - + // should return prior token fs.toUrl(GetOpParam.Op.OPEN, null); verify(fs).getDelegationToken(); verify(fs, never()).getDelegationToken(null); verify(fs, never()).setDelegationToken(token); - verify(fs, never()).addRenewAction(fs); } - - @Test(timeout=1000) + + @Test(timeout = 1000) public void testNoTokenForGetToken() throws IOException { checkNoTokenForOperation(GetOpParam.Op.GETDELEGATIONTOKEN); } - - @Test(timeout=1000) + + @Test(timeout = 1000) public void testNoTokenForCanclToken() throws IOException { checkNoTokenForOperation(PutOpParam.Op.RENEWDELEGATIONTOKEN); } - @Test(timeout=1000) + @Test(timeout = 1000) public void testNoTokenForCancelToken() throws IOException { checkNoTokenForOperation(PutOpParam.Op.CANCELDELEGATIONTOKEN); } @SuppressWarnings("unchecked") private void checkNoTokenForOperation(HttpOpParam.Op op) throws IOException { - WebHdfsFileSystem fs = spy(new WebHdfsFileSystem()); - doReturn(null).when(fs).selectDelegationToken(ugi); + WebHdfsFileSystem fs = spyWebhdfsInSecureSetup(); doReturn(null).when(fs).getDelegationToken(null); - doNothing().when(fs).addRenewAction(any(WebHdfsFileSystem.class)); fs.initialize(URI.create("webhdfs://127.0.0.1:0"), conf); - + // do not get a token! fs.toUrl(op, null); verify(fs, never()).getDelegationToken(); verify(fs, never()).getDelegationToken(null); verify(fs, never()).setDelegationToken(any(Token.class)); - verify(fs, never()).addRenewAction(fs); } - - @Test(timeout=1000) + + @Test(timeout = 1000) public void testGetOpRequireAuth() { for (HttpOpParam.Op op : GetOpParam.Op.values()) { boolean expect = (op == GetOpParam.Op.GETDELEGATIONTOKEN); - assertEquals(expect, op.getRequireAuth()); + assertEquals(expect, op.getRequireAuth()); } } - @Test(timeout=1000) + @Test(timeout = 1000) public void testPutOpRequireAuth() { for (HttpOpParam.Op op : PutOpParam.Op.values()) { - boolean expect = (op == PutOpParam.Op.RENEWDELEGATIONTOKEN || - op == PutOpParam.Op.CANCELDELEGATIONTOKEN); - assertEquals(expect, op.getRequireAuth()); + boolean expect = (op == PutOpParam.Op.RENEWDELEGATIONTOKEN || op == PutOpParam.Op.CANCELDELEGATIONTOKEN); + assertEquals(expect, op.getRequireAuth()); } } - - @Test(timeout=1000) - public void testPostOpRequireAuth() { + + @Test(timeout = 1000) + public void testPostOpRequireAuth() { for (HttpOpParam.Op op : PostOpParam.Op.values()) { assertFalse(op.getRequireAuth()); } } - - @Test(timeout=1000) - public void testDeleteOpRequireAuth() { + + @Test(timeout = 1000) + public void testDeleteOpRequireAuth() { for (HttpOpParam.Op op : DeleteOpParam.Op.values()) { assertFalse(op.getRequireAuth()); } } - - @Test - public void testGetTokenAfterFailure() throws Exception { - Configuration conf = mock(Configuration.class); - Token token1 = mock(Token.class); - Token token2 = mock(Token.class); - long renewCycle = 1000; - - DelegationTokenRenewer.renewCycle = renewCycle; - WebHdfsFileSystem fs = spy(new WebHdfsFileSystem()); - doReturn(conf).when(fs).getConf(); - doReturn(token1).doReturn(token2).when(fs).getDelegationToken(null); - // cause token renewer to abandon the token - doThrow(new IOException("renew failed")).when(token1).renew(conf); - doThrow(new IOException("get failed")).when(fs).addDelegationTokens(null, null); - - // trigger token acquisition - Token token = fs.getDelegationToken(); - RenewAction action = fs.action; - assertSame(token1, token); - assertTrue(action.isValid()); - - // fetch again and make sure it's the same as before - token = fs.getDelegationToken(); - assertSame(token1, token); - assertSame(action, fs.action); - assertTrue(fs.action.isValid()); - - // upon renewal, token will go bad based on above stubbing - Thread.sleep(renewCycle); - assertSame(action, fs.action); - assertFalse(fs.action.isValid()); - - // now that token is invalid, should get a new one - token = fs.getDelegationToken(); - assertSame(token2, token); - assertNotSame(action, fs.action); - assertTrue(fs.action.isValid()); - action = fs.action; - - // should get same one again - token = fs.getDelegationToken(); - assertSame(token2, token); - assertSame(action, fs.action); - assertTrue(fs.action.isValid()); - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java index 2bcab6ef6f..a84918e13d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java @@ -20,8 +20,6 @@ import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.mockito.Mockito.mock; import java.io.IOException; @@ -36,15 +34,20 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.web.resources.*; +import org.apache.hadoop.hdfs.web.resources.DelegationParam; +import org.apache.hadoop.hdfs.web.resources.DoAsParam; +import org.apache.hadoop.hdfs.web.resources.GetOpParam; +import org.apache.hadoop.hdfs.web.resources.PutOpParam; +import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam; +import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.SecurityUtilTestHelper; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.junit.*; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; public class TestWebHdfsUrl { // NOTE: port is never used @@ -306,95 +309,4 @@ private WebHdfsFileSystem getWebHdfsFileSystem(UserGroupInformation ugi, } return (WebHdfsFileSystem) FileSystem.get(uri, conf); } - - @Test(timeout=60000) - public void testSelectHdfsDelegationToken() throws Exception { - SecurityUtilTestHelper.setTokenServiceUseIp(true); - - Configuration conf = new Configuration(); - conf.setClass("fs.webhdfs.impl", MyWebHdfsFileSystem.class, FileSystem.class); - - // test with implicit default port - URI fsUri = URI.create("webhdfs://localhost"); - MyWebHdfsFileSystem fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); - - // test with explicit default port - fsUri = URI.create("webhdfs://localhost:"+fs.getDefaultPort()); - fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); - - // test with non-default port - fsUri = URI.create("webhdfs://localhost:"+(fs.getDefaultPort()-1)); - fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); - - } - - private void checkTokenSelection(MyWebHdfsFileSystem fs, - Configuration conf) throws IOException { - int port = fs.getCanonicalUri().getPort(); - // can't clear tokens from ugi, so create a new user everytime - UserGroupInformation ugi = - UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{}); - - // use ip-based tokens - SecurityUtilTestHelper.setTokenServiceUseIp(true); - - // test fallback to hdfs token - Token hdfsToken = new Token( - new byte[0], new byte[0], - DelegationTokenIdentifier.HDFS_DELEGATION_KIND, - new Text("127.0.0.1:8020")); - ugi.addToken(hdfsToken); - - // test fallback to hdfs token - Token token = fs.selectDelegationToken(ugi); - assertNotNull(token); - assertEquals(hdfsToken, token); - - // test webhdfs is favored over hdfs - Token webHdfsToken = new Token( - new byte[0], new byte[0], - WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port)); - ugi.addToken(webHdfsToken); - token = fs.selectDelegationToken(ugi); - assertNotNull(token); - assertEquals(webHdfsToken, token); - - // switch to using host-based tokens, no token should match - SecurityUtilTestHelper.setTokenServiceUseIp(false); - token = fs.selectDelegationToken(ugi); - assertNull(token); - - // test fallback to hdfs token - hdfsToken = new Token( - new byte[0], new byte[0], - DelegationTokenIdentifier.HDFS_DELEGATION_KIND, - new Text("localhost:8020")); - ugi.addToken(hdfsToken); - token = fs.selectDelegationToken(ugi); - assertNotNull(token); - assertEquals(hdfsToken, token); - - // test webhdfs is favored over hdfs - webHdfsToken = new Token( - new byte[0], new byte[0], - WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:"+port)); - ugi.addToken(webHdfsToken); - token = fs.selectDelegationToken(ugi); - assertNotNull(token); - assertEquals(webHdfsToken, token); - } - - static class MyWebHdfsFileSystem extends WebHdfsFileSystem { - @Override - public URI getCanonicalUri() { - return super.getCanonicalUri(); - } - @Override - public int getDefaultPort() { - return super.getDefaultPort(); - } - } }