diff --git a/CHANGES.txt b/CHANGES.txt index 0145750412..6d594598b3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -76,6 +76,9 @@ Trunk (unreleased changes) HADOOP-6435. Make RPC.waitForProxy with timeout public. (Steve Loughran via tomwhite) + + HADOOP-6472. add tokenCache option to GenericOptionsParser for passing + file with secret keys to a map reduce job. (boryas) OPTIMIZATIONS diff --git a/src/java/org/apache/hadoop/util/GenericOptionsParser.java b/src/java/org/apache/hadoop/util/GenericOptionsParser.java index dc13feddf9..46973c28dd 100644 --- a/src/java/org/apache/hadoop/util/GenericOptionsParser.java +++ b/src/java/org/apache/hadoop/util/GenericOptionsParser.java @@ -233,6 +233,12 @@ private static Options buildGeneralOptions(Options opts) { .withDescription("comma separated archives to be unarchived" + " on the compute machines.") .create("archives"); + + // file with security tokens + Option tokensFile = OptionBuilder.withArgName("tokensFile") + .hasArg() + .withDescription("name of the file with the tokens") + .create("tokenCacheFile"); opts.addOption(fs); opts.addOption(jt); @@ -241,6 +247,7 @@ private static Options buildGeneralOptions(Options opts) { opts.addOption(libjars); opts.addOption(files); opts.addOption(archives); + opts.addOption(tokensFile); return opts; } @@ -295,6 +302,19 @@ private void processGeneralOptions(Configuration conf, } } conf.setBoolean("mapred.used.genericoptionsparser", true); + + // tokensFile + if(line.hasOption("tokenCacheFile")) { + String fileName = line.getOptionValue("tokenCacheFile"); + // check if the local file exists + FileSystem localFs = FileSystem.getLocal(conf); + Path p = new Path(fileName); + if (!localFs.exists(p)) { + throw new FileNotFoundException("File "+fileName+" does not exist."); + } + LOG.debug("setting conf tokensFile: " + fileName); + conf.set("tokenCacheFile", localFs.makeQualified(p).toString()); + } } /** diff --git a/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java b/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java index 5b48db1a58..a9474c8813 100644 --- a/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java +++ b/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java @@ -19,22 +19,23 @@ import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import java.net.URI; +import junit.framework.TestCase; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import junit.framework.TestCase; - public class TestGenericOptionsParser extends TestCase { - private static File testDir = - new File(System.getProperty("test.build.data", "/tmp"), "generic"); + File testDir; + Configuration conf; + FileSystem localFs; + public void testFilesOption() throws Exception { - Configuration conf = new Configuration(); File tmpFile = new File(testDir, "tmpfile"); - FileSystem localFs = FileSystem.getLocal(conf); Path tmpPath = new Path(tmpFile.toString()); localFs.create(tmpPath); String[] args = new String[2]; @@ -74,7 +75,62 @@ public void testFilesOption() throws Exception { th instanceof FileNotFoundException); files = conf2.get("tmpfiles"); assertNull("files is not null", files); - testDir.delete(); + } + + @Override + protected void setUp() throws Exception { + super.setUp(); + conf = new Configuration(); + localFs = FileSystem.getLocal(conf); + testDir = new File(System.getProperty("test.build.data", "/tmp"), "generic"); + if(testDir.exists()) + localFs.delete(new Path(testDir.toString()), true); } + @Override + protected void tearDown() throws Exception { + super.tearDown(); + if(testDir.exists()) { + localFs.delete(new Path(testDir.toString()), true); + } + } + + /** + * testing -fileCache option + * @throws IOException + */ + public void testTokenCacheOption() throws IOException { + FileSystem localFs = FileSystem.getLocal(conf); + + File tmpFile = new File(testDir, "tokenCacheFile"); + if(tmpFile.exists()) { + tmpFile.delete(); + } + String[] args = new String[2]; + // pass a files option + args[0] = "-tokenCacheFile"; + args[1] = tmpFile.toString(); + + // test non existing file + Throwable th = null; + try { + new GenericOptionsParser(conf, args); + } catch (Exception e) { + th = e; + } + assertNotNull(th); + assertTrue("FileNotFoundException is not thrown", + th instanceof FileNotFoundException); + + // create file + Path tmpPath = new Path(tmpFile.toString()); + localFs.create(tmpPath); + new GenericOptionsParser(conf, args); + String fileName = conf.get("tokenCacheFile"); + assertNotNull("files is null", fileName); + assertEquals("files option does not match", + localFs.makeQualified(tmpPath).toString(), fileName); + + localFs.delete(new Path(testDir.getAbsolutePath()), true); + } }