From aea890f7d215d97feec873228158daefa2e63217 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Fri, 11 May 2012 15:00:48 +0000 Subject: [PATCH] MAPREDUCE-4148. MapReduce should not have a compile-time dependency on HDFS. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1337199 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/security/token/Token.java | 70 +++++++++++++++++-- .../hadoop/security/token/TestToken.java | 19 +++++ ...ache.hadoop.security.token.TokenIdentifier | 2 + ...ache.hadoop.security.token.TokenIdentifier | 2 + hadoop-mapreduce-project/CHANGES.txt | 3 + ...ache.hadoop.security.token.TokenIdentifier | 1 + .../hadoop-mapreduce-client-core/pom.xml | 1 + .../apache/hadoop/mapreduce/JobSubmitter.java | 4 +- .../hadoop/mapreduce/security/TokenCache.java | 7 +- .../token/DelegationTokenRenewal.java | 1 - ...ache.hadoop.security.token.TokenIdentifier | 2 + .../hadoop-mapreduce-client/pom.xml | 2 +- .../hadoop-mapreduce-examples/pom.xml | 2 +- ...ache.hadoop.security.token.TokenIdentifier | 4 ++ ...ache.hadoop.security.token.TokenIdentifier | 1 + hadoop-mapreduce-project/pom.xml | 2 +- 16 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java index e95ade860b..bbddf6fdc7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java @@ -18,10 +18,15 @@ package org.apache.hadoop.security.token; +import com.google.common.collect.Maps; + +import java.io.ByteArrayInputStream; import java.io.DataInput; +import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; import java.util.Arrays; +import java.util.Map; import java.util.ServiceLoader; import org.apache.commons.codec.binary.Base64; @@ -37,6 +42,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.util.ReflectionUtils; /** * The client-side form of the token. @@ -45,6 +51,9 @@ @InterfaceStability.Evolving public class Token implements Writable { public static final Log LOG = LogFactory.getLog(Token.class); + + private static Map> tokenKindMap; + private byte[] identifier; private byte[] password; private Text kind; @@ -100,13 +109,49 @@ public Token(Token other) { } /** - * Get the token identifier - * @return the token identifier + * Get the token identifier's byte representation + * @return the token identifier's byte representation */ public byte[] getIdentifier() { return identifier; } + private static synchronized Class + getClassForIdentifier(Text kind) { + if (tokenKindMap == null) { + tokenKindMap = Maps.newHashMap(); + for (TokenIdentifier id : ServiceLoader.load(TokenIdentifier.class)) { + tokenKindMap.put(id.getKind(), id.getClass()); + } + } + Class cls = tokenKindMap.get(kind); + if (cls == null) { + LOG.warn("Cannot find class for token kind " + kind); + return null; + } + return cls; + } + + /** + * Get the token identifier object, or null if it could not be constructed + * (because the class could not be loaded, for example). + * @return the token identifier, or null + * @throws IOException + */ + @SuppressWarnings("unchecked") + public T decodeIdentifier() throws IOException { + Class cls = getClassForIdentifier(getKind()); + if (cls == null) { + return null; + } + TokenIdentifier tokenIdentifier = ReflectionUtils.newInstance(cls, null); + ByteArrayInputStream buf = new ByteArrayInputStream(identifier); + DataInputStream in = new DataInputStream(buf); + tokenIdentifier.readFields(in); + in.close(); + return (T) tokenIdentifier; + } + /** * Get the token password/secret * @return the token password/secret @@ -260,16 +305,31 @@ private static void addBinaryBuffer(StringBuilder buffer, byte[] bytes) { buffer.append(num); } } + + private void identifierToString(StringBuilder buffer) { + T id = null; + try { + id = decodeIdentifier(); + } catch (IOException e) { + // handle in the finally block + } finally { + if (id != null) { + buffer.append("(").append(id).append(")"); + } else { + addBinaryBuffer(buffer, identifier); + } + } + } @Override public String toString() { StringBuilder buffer = new StringBuilder(); - buffer.append("Ident: "); - addBinaryBuffer(buffer, identifier); - buffer.append(", Kind: "); + buffer.append("Kind: "); buffer.append(kind.toString()); buffer.append(", Service: "); buffer.append(service.toString()); + buffer.append(", Ident: "); + identifierToString(buffer); return buffer.toString(); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestToken.java index 54b75da23b..6d7d695663 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestToken.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestToken.java @@ -18,11 +18,15 @@ package org.apache.hadoop.security.token; +import static junit.framework.Assert.assertEquals; + import java.io.*; import java.util.Arrays; import org.apache.hadoop.io.*; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.TestDelegationToken.TestDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.TestDelegationToken.TestDelegationTokenSecretManager; import junit.framework.TestCase; @@ -94,5 +98,20 @@ public static void testEncodeWritable() throws Exception { checkUrlSafe(encode); } } + + public void testDecodeIdentifier() throws IOException { + TestDelegationTokenSecretManager secretManager = + new TestDelegationTokenSecretManager(0, 0, 0, 0); + secretManager.startThreads(); + TestDelegationTokenIdentifier id = new TestDelegationTokenIdentifier( + new Text("owner"), new Text("renewer"), new Text("realUser")); + + Token token = + new Token(id, secretManager); + TokenIdentifier idCopy = token.decodeIdentifier(); + + assertNotSame(id, idCopy); + assertEquals(id, idCopy); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier new file mode 100644 index 0000000000..891a67b61f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -0,0 +1,2 @@ +org.apache.hadoop.ipc.TestSaslRPC$TestTokenIdentifier +org.apache.hadoop.security.token.delegation.TestDelegationToken$TestDelegationTokenIdentifier diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier new file mode 100644 index 0000000000..10b874b685 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -0,0 +1,2 @@ +org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier +org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 7b8e045a2d..0be7073a9a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -287,6 +287,9 @@ Release 2.0.0 - UNRELEASED MAPREDUCE-4231. Update RAID to use the new BlockCollection interface. (szetszwo) + MAPREDUCE-4148. MapReduce should not have a compile-time dependency on + HDFS. (tomwhite) + Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier new file mode 100644 index 0000000000..0975deab7e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -0,0 +1 @@ +org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index cfb8ce4bd7..e60d745faa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -37,6 +37,7 @@ org.apache.hadoop hadoop-hdfs + test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java index 4038f65cd4..148df50324 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java @@ -38,7 +38,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.QueueACL; @@ -433,8 +432,7 @@ private void printTokens(JobID jobId, LOG.debug("Printing tokens for job: " + jobId); for(Token token: credentials.getAllTokens()) { if (token.getKind().toString().equals("HDFS_DELEGATION_TOKEN")) { - LOG.debug("Submitting with " + - org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier.stringifyToken(token)); + LOG.debug("Submitting with " + token); } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index 9e8c1909a1..1109f3f382 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -30,7 +30,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Master; @@ -179,16 +178,14 @@ private static void mergeBinaryTokens(Credentials creds, Configuration conf) { * @param namenode * @return delegation token */ - @SuppressWarnings("unchecked") @InterfaceAudience.Private - public static Token getDelegationToken( + public static Token getDelegationToken( Credentials credentials, String namenode) { //No fs specific tokens issues by this fs. It may however issue tokens // for other filesystems - which would be keyed by that filesystems name. if (namenode == null) return null; - return (Token) credentials.getToken(new Text( - namenode)); + return (Token) credentials.getToken(new Text(namenode)); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java index e4675b523a..9000777069 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java @@ -39,7 +39,6 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.util.StringUtils; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier new file mode 100644 index 0000000000..f797a6aa6f --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -0,0 +1,2 @@ +org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier +org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index 5a96b31539..3cff07d133 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -114,8 +114,8 @@ org.apache.hadoop - hadoop-hdfs + test com.google.inject.extensions diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml index 3520839173..f02617f467 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml @@ -57,7 +57,7 @@ org.apache.hadoop hadoop-hdfs - provided + runtime org.apache.hadoop diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier new file mode 100644 index 0000000000..fc669de157 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -0,0 +1,4 @@ +org.apache.hadoop.yarn.security.ContainerTokenIdentifier +org.apache.hadoop.yarn.security.ApplicationTokenIdentifier +org.apache.hadoop.yarn.security.client.ClientTokenIdentifier +org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier new file mode 100644 index 0000000000..6ed6e3261e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -0,0 +1 @@ +org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenIdentifier diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index cca8427355..3b340fddbb 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -128,8 +128,8 @@ org.apache.hadoop - hadoop-hdfs + test com.google.inject