From c15ae29a906cd2ee8ee526cc67f1cc992c673534 Mon Sep 17 00:00:00 2001 From: Boris Shkolnik Date: Fri, 30 Jul 2010 04:55:23 +0000 Subject: [PATCH] HADOOP-6873. using delegation token over hftp for long running clients git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@980648 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 3 ++ .../apache/hadoop/fs/FSDataInputStream.java | 2 +- src/java/org/apache/hadoop/fs/FileSystem.java | 29 ++++++++++++++++++- .../apache/hadoop/security/Credentials.java | 29 ++++++++++--------- .../apache/hadoop/security/SecurityUtil.java | 8 ++++- .../hadoop/security/UserGroupInformation.java | 4 +-- .../hadoop/fs/TestFilterFileSystem.java | 11 ++++++- 7 files changed, 67 insertions(+), 19 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 70f04a71cf..261222f97c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -166,6 +166,9 @@ Trunk (unreleased changes) HADOOP-6536. Fixes FileUtil.fullyDelete() not to delete the contents of the sym-linked directory. (Ravi Gummadi via amareshwari) + HADOOP-6873. using delegation token over hftp for long + running clients (boryas) + Release 0.21.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/src/java/org/apache/hadoop/fs/FSDataInputStream.java b/src/java/org/apache/hadoop/fs/FSDataInputStream.java index 4a4eb447d8..cb4dd61470 100644 --- a/src/java/org/apache/hadoop/fs/FSDataInputStream.java +++ b/src/java/org/apache/hadoop/fs/FSDataInputStream.java @@ -27,7 +27,7 @@ @InterfaceAudience.Public @InterfaceStability.Stable public class FSDataInputStream extends DataInputStream - implements Seekable, PositionedReadable { + implements Seekable, PositionedReadable, Closeable { public FSDataInputStream(InputStream in) throws IOException { diff --git a/src/java/org/apache/hadoop/fs/FileSystem.java b/src/java/org/apache/hadoop/fs/FileSystem.java index 9a994e3556..812195cf34 100644 --- a/src/java/org/apache/hadoop/fs/FileSystem.java +++ b/src/java/org/apache/hadoop/fs/FileSystem.java @@ -47,9 +47,10 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.MultipleIOException; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; @@ -169,6 +170,22 @@ public void initialize(URI name, Configuration conf) throws IOException { /** Returns a URI whose scheme and authority identify this FileSystem.*/ public abstract URI getUri(); + /** + * Get the default port for this file system. + * @return the default port or 0 if there isn't one + */ + protected int getDefaultPort() { + return 0; + } + + /** + * Get a canonical name for this file system. + * @return a URI string that uniquely identifies this file system + */ + public String getCanonicalServiceName() { + return SecurityUtil.buildDTServiceName(getUri(), getDefaultPort()); + } + /** @deprecated call #getUri() instead.*/ @Deprecated public String getName() { return getUri().toString(); } @@ -328,6 +345,16 @@ public Path makeQualified(Path path) { return path.makeQualified(this.getUri(), this.getWorkingDirectory()); } + /** + * Get a new delegation token for this file system. + * @param renewer the account name that is allowed to renew the token. + * @return a new delegation token + * @throws IOException + */ + public Token getDelegationToken(String renewer) throws IOException { + return null; + } + /** create a file with the provided permission * The permission of the file is set to be the provided permission as in * setPermission, not permission&~umask diff --git a/src/java/org/apache/hadoop/security/Credentials.java b/src/java/org/apache/hadoop/security/Credentials.java index 6df4be798a..152a35496b 100644 --- a/src/java/org/apache/hadoop/security/Credentials.java +++ b/src/java/org/apache/hadoop/security/Credentials.java @@ -28,20 +28,20 @@ import java.util.HashMap; import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; /** * A class that provides the facilities of reading and writing @@ -124,15 +124,18 @@ public void addSecretKey(Text alias, byte[] key) { * @param conf * @throws IOException */ - public void readTokenStorageFile(Path filename, - Configuration conf) throws IOException { - FSDataInputStream in = filename.getFileSystem(conf).open(filename); + public static Credentials readTokenStorageFile(Path filename, Configuration conf) + throws IOException { + FSDataInputStream in = null; + Credentials credentials = new Credentials(); try { - readTokenStorageStream(in); - } catch(IOException ioe) { - throw new IOException("Exception reading " + filename, ioe); - } finally { + in = filename.getFileSystem(conf).open(filename); + credentials.readTokenStorageStream(in); in.close(); + return credentials; + } catch(IOException ioe) { + IOUtils.cleanup(LOG, in); + throw new IOException("Exception reading " + filename, ioe); } } diff --git a/src/java/org/apache/hadoop/security/SecurityUtil.java b/src/java/org/apache/hadoop/security/SecurityUtil.java index 5802474f6a..00187bd6f2 100644 --- a/src/java/org/apache/hadoop/security/SecurityUtil.java +++ b/src/java/org/apache/hadoop/security/SecurityUtil.java @@ -234,7 +234,13 @@ public static String buildDTServiceName(URI uri, int defPort) { // for whatever reason using NetUtils.createSocketAddr(target).toString() // returns "localhost/ip:port" StringBuffer sb = new StringBuffer(); - sb.append(NetUtils.normalizeHostName(uri.getHost())).append(":").append(port); + String host = uri.getHost(); + if (host != null) { + host = NetUtils.normalizeHostName(host); + } else { + host = ""; + } + sb.append(host).append(":").append(port); return sb.toString(); } } diff --git a/src/java/org/apache/hadoop/security/UserGroupInformation.java b/src/java/org/apache/hadoop/security/UserGroupInformation.java index 56110d9e58..2d8df5c6c1 100644 --- a/src/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/src/java/org/apache/hadoop/security/UserGroupInformation.java @@ -415,8 +415,8 @@ static UserGroupInformation getLoginUser() throws IOException { if (fileLocation != null && isSecurityEnabled()) { // load the token storage file and put all of the tokens into the // user. - Credentials cred = new Credentials(); - cred.readTokenStorageFile(new Path("file:///" + fileLocation), conf); + Credentials cred = Credentials.readTokenStorageFile( + new Path("file:///" + fileLocation), conf); for (Token token: cred.getAllTokens()) { loginUser.addToken(token); } diff --git a/src/test/core/org/apache/hadoop/fs/TestFilterFileSystem.java b/src/test/core/org/apache/hadoop/fs/TestFilterFileSystem.java index bab022d215..24c6de2612 100644 --- a/src/test/core/org/apache/hadoop/fs/TestFilterFileSystem.java +++ b/src/test/core/org/apache/hadoop/fs/TestFilterFileSystem.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs; +import java.io.IOException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.EnumSet; @@ -28,6 +29,8 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.Rename; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; public class TestFilterFileSystem extends TestCase { @@ -123,7 +126,13 @@ public FSDataOutputStream primitiveCreate(final Path f, CreateOpts... opts) { return null; } public void primitiveMkdir(Path f, FsPermission absolutePermission, boolean createParent) { } - } + public int getDefaultPort() { return 0; } + public String getCanonicalServiceName() { return null; } + public Token getDelegationToken(String renewer) throws IOException { + return null; + } + + } public void testFilterFileSystem() throws Exception { for (Method m : FileSystem.class.getDeclaredMethods()) {