diff --git a/CHANGES.txt b/CHANGES.txt index 56b3a9d688..062fae3516 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -80,6 +80,9 @@ Trunk (unreleased changes) same principal. Now the principal name is a pattern that has _HOST in it. (Kan Zhang & Jitendra Pandey via ddas) + HADOOP-6861. Adds new non-static methods in Credentials to read and + write token storage file. (Jitendra Pandey & Owen O'Malley via ddas) + OPTIMIZATIONS BUG FIXES diff --git a/src/java/org/apache/hadoop/io/WritableUtils.java b/src/java/org/apache/hadoop/io/WritableUtils.java index 575c967934..e0278b8aaf 100644 --- a/src/java/org/apache/hadoop/io/WritableUtils.java +++ b/src/java/org/apache/hadoop/io/WritableUtils.java @@ -57,7 +57,8 @@ public static void skipCompressedByteArray(DataInput in) throws IOException { } } - public static int writeCompressedByteArray(DataOutput out, byte[] bytes) throws IOException { + public static int writeCompressedByteArray(DataOutput out, + byte[] bytes) throws IOException { if (bytes != null) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); GZIPOutputStream gzout = new GZIPOutputStream(bos); diff --git a/src/java/org/apache/hadoop/security/Credentials.java b/src/java/org/apache/hadoop/security/Credentials.java index 6b13daa615..6df4be798a 100644 --- a/src/java/org/apache/hadoop/security/Credentials.java +++ b/src/java/org/apache/hadoop/security/Credentials.java @@ -19,13 +19,17 @@ package org.apache.hadoop.security; import java.io.DataInput; +import java.io.DataInputStream; import java.io.DataOutput; +import java.io.DataOutputStream; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -114,24 +118,59 @@ public void addSecretKey(Text alias, byte[] key) { } /** - * Convenience method for reading a file, and loading the Tokens + * Convenience method for reading a token storage file, and loading the Tokens * therein in the passed UGI * @param filename * @param conf - * @param ugi * @throws IOException */ - public static void readTokensAndLoadInUGI(String filename, Configuration conf, - UserGroupInformation ugi) throws IOException { - Path localTokensFile = new Path (filename); - FileSystem localFS = FileSystem.getLocal(conf); - FSDataInputStream in = localFS.open(localTokensFile); - Credentials ts = new Credentials(); - ts.readFields(in); - for (Token token : ts.getAllTokens()) { - ugi.addToken(token); + public void readTokenStorageFile(Path filename, + Configuration conf) throws IOException { + FSDataInputStream in = filename.getFileSystem(conf).open(filename); + try { + readTokenStorageStream(in); + } catch(IOException ioe) { + throw new IOException("Exception reading " + filename, ioe); + } finally { + in.close(); } } + + /** + * Convenience method for reading a token storage file directly from a + * datainputstream + */ + public void readTokenStorageStream(DataInputStream in) throws IOException { + byte[] magic = new byte[TOKEN_STORAGE_MAGIC.length]; + in.readFully(magic); + if (!Arrays.equals(magic, TOKEN_STORAGE_MAGIC)) { + throw new IOException("Bad header found in token storage."); + } + byte version = in.readByte(); + if (version != TOKEN_STORAGE_VERSION) { + throw new IOException("Unknown version " + version + + " in token storage."); + } + readFields(in); + } + + private static final byte[] TOKEN_STORAGE_MAGIC = "HDTS".getBytes(); + private static final byte TOKEN_STORAGE_VERSION = 0; + + public void writeTokenStorageToStream(DataOutputStream os) + throws IOException { + os.write(TOKEN_STORAGE_MAGIC); + os.write(TOKEN_STORAGE_VERSION); + write(os); + } + + public void writeTokenStorageFile(Path filename, + Configuration conf) throws IOException { + FSDataOutputStream os = filename.getFileSystem(conf).create(filename); + writeTokenStorageToStream(os); + os.close(); + } + /** * Stores all the keys to DataOutput * @param out @@ -151,7 +190,8 @@ public void write(DataOutput out) throws IOException { WritableUtils.writeVInt(out, secretKeysMap.size()); for(Map.Entry e : secretKeysMap.entrySet()) { e.getKey().write(out); - WritableUtils.writeCompressedByteArray(out, e.getValue()); + WritableUtils.writeVInt(out, e.getValue().length); + out.write(e.getValue()); } } @@ -178,8 +218,23 @@ public void readFields(DataInput in) throws IOException { for(int i=0; i secret: other.secretKeysMap.entrySet()) { + secretKeysMap.put(secret.getKey(), secret.getValue()); + } + for(Map.Entry> token: other.tokenMap.entrySet()){ + tokenMap.put(token.getKey(), token.getValue()); } } } diff --git a/src/java/org/apache/hadoop/security/UserGroupInformation.java b/src/java/org/apache/hadoop/security/UserGroupInformation.java index a0940ed80b..8eed66b454 100644 --- a/src/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/src/java/org/apache/hadoop/security/UserGroupInformation.java @@ -50,6 +50,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -135,6 +136,9 @@ public boolean logout() throws LoginException { private static boolean useKerberos; /** Server-side groups fetching service */ private static Groups groups; + /** The configuration to use */ + private static Configuration conf; + public static final long MIN_TIME_BEFORE_RELOGIN = 10 * 60 * 1000L; @@ -188,6 +192,7 @@ private static synchronized void initialize(Configuration conf) { "configuration", ioe); } isInitialized = true; + UserGroupInformation.conf = conf; } /** @@ -398,9 +403,15 @@ static UserGroupInformation getLoginUser() throws IOException { login.login(); loginUser.setLogin(login); loginUser = new UserGroupInformation(login.getSubject()); - String tokenFile = System.getenv(HADOOP_TOKEN_FILE_LOCATION); - if (tokenFile != null && isSecurityEnabled()) { - Credentials.readTokensAndLoadInUGI(tokenFile, new Configuration(), loginUser); + String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION); + if (fileLocation != null && isSecurityEnabled()) { + // load the token storage file and put all of the tokens into the + // user. + Credentials cred = new Credentials(); + cred.readTokenStorageFile(new Path("file:///" + fileLocation), conf); + for (Token token: cred.getAllTokens()) { + loginUser.addToken(token); + } } } catch (LoginException le) { throw new IOException("failure to login", le); diff --git a/src/java/org/apache/hadoop/util/GenericOptionsParser.java b/src/java/org/apache/hadoop/util/GenericOptionsParser.java index 4113e81ff0..c22400704c 100644 --- a/src/java/org/apache/hadoop/util/GenericOptionsParser.java +++ b/src/java/org/apache/hadoop/util/GenericOptionsParser.java @@ -317,7 +317,9 @@ private void processGeneralOptions(Configuration conf, throw new FileNotFoundException("File "+fileName+" does not exist."); } LOG.debug("setting conf tokensFile: " + fileName); - conf.set("tokenCacheFile", localFs.makeQualified(p).toString()); + conf.set("mapreduce.job.credentials.json", localFs.makeQualified(p) + .toString()); + } } diff --git a/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java b/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java index a9474c8813..bcabcfa64d 100644 --- a/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java +++ b/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java @@ -126,7 +126,7 @@ public void testTokenCacheOption() throws IOException { Path tmpPath = new Path(tmpFile.toString()); localFs.create(tmpPath); new GenericOptionsParser(conf, args); - String fileName = conf.get("tokenCacheFile"); + String fileName = conf.get("mapreduce.job.credentials.json"); assertNotNull("files is null", fileName); assertEquals("files option does not match", localFs.makeQualified(tmpPath).toString(), fileName);