diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0ee6c90693..8e526785e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -273,6 +273,9 @@ Release 2.0.1-alpha - UNRELEASED HDFS-3484. hdfs fsck doesn't work if NN HTTP address is set to 0.0.0.0 even if NN RPC address is configured. (atm via eli) + HDFS-3486. offlineimageviewer can't read fsimage files that contain + persistent delegation tokens. (Colin Patrick McCabe via eli) + Release 2.0.0-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java index fdc9892e1a..3a6ce3ef1a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java @@ -31,11 +31,13 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; import org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor.ImageElement; +import org.apache.hadoop.hdfs.util.XMLUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.xml.sax.helpers.AttributesImpl; /** * ImageLoaderCurrent processes Hadoop FSImage files and walks over @@ -220,9 +222,29 @@ private void processDelegationTokens(DataInputStream in, ImageVisitor v) for(int i=0; i writtenFiles - = new HashMap(); - + final static HashMap writtenFiles = + new HashMap(); private static String ROOT = System.getProperty("test.build.data", "build/test/data"); - // Main entry point into testing. Necessary since we only want to generate - // the fsimage file once and use it for multiple tests. - public void testOIV() throws Exception { - File originalFsimage = null; - try { - originalFsimage = initFsimage(); - assertNotNull("originalFsImage shouldn't be null", originalFsimage); - - // Tests: - outputOfLSVisitor(originalFsimage); - outputOfFileDistributionVisitor(originalFsimage); - - unsupportedFSLayoutVersion(originalFsimage); - - truncatedFSImage(originalFsimage); - - } finally { - if(originalFsimage != null && originalFsimage.exists()) - originalFsimage.delete(); - } - } - // Create a populated namespace for later testing. Save its contents to a // data structure and store its fsimage location. - private File initFsimage() throws IOException { + // We only want to generate the fsimage file once and use it for + // multiple tests. + @BeforeClass + public static void createOriginalFSImage() throws IOException { MiniDFSCluster cluster = null; - File orig = null; try { Configuration conf = new HdfsConfiguration(); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + conf.set("hadoop.security.auth_to_local", + "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + cluster.waitActive(); FileSystem hdfs = cluster.getFileSystem(); int filesize = 256; @@ -123,34 +119,49 @@ private File initFsimage() throws IOException { } } + // Get delegation tokens so we log the delegation token op + List> delegationTokens = + hdfs.getDelegationTokens(TEST_RENEWER); + for (Token t : delegationTokens) { + LOG.debug("got token " + t); + } + // Write results to the fsimage file cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER); cluster.getNameNodeRpc().saveNamespace(); // Determine location of fsimage file - orig = FSImageTestUtil.findLatestImageFile( + originalFsimage = FSImageTestUtil.findLatestImageFile( FSImageTestUtil.getFSImage( cluster.getNameNode()).getStorage().getStorageDir(0)); - if (orig == null) { - fail("Didn't generate or can't find fsimage"); + if (originalFsimage == null) { + throw new RuntimeException("Didn't generate or can't find fsimage"); } + LOG.debug("original FS image file is " + originalFsimage); } finally { if(cluster != null) cluster.shutdown(); } - return orig; + } + + @AfterClass + public static void deleteOriginalFSImage() throws IOException { + if(originalFsimage != null && originalFsimage.exists()) { + originalFsimage.delete(); + } } // Convenience method to generate a file status from file system for // later comparison - private FileStatus pathToFileEntry(FileSystem hdfs, String file) + private static FileStatus pathToFileEntry(FileSystem hdfs, String file) throws IOException { return hdfs.getFileStatus(new Path(file)); } - + // Verify that we can correctly generate an ls-style output for a valid // fsimage - private void outputOfLSVisitor(File originalFsimage) throws IOException { + @Test + public void outputOfLSVisitor() throws IOException { File testFile = new File(ROOT, "/basicCheck"); File outputFile = new File(ROOT, "/basicCheckOutput"); @@ -169,12 +180,13 @@ private void outputOfLSVisitor(File originalFsimage) throws IOException { if(testFile.exists()) testFile.delete(); if(outputFile.exists()) outputFile.delete(); } - System.out.println("Correctly generated ls-style output."); + LOG.debug("Correctly generated ls-style output."); } // Confirm that attempting to read an fsimage file with an unsupported // layout results in an error - public void unsupportedFSLayoutVersion(File originalFsimage) throws IOException { + @Test + public void unsupportedFSLayoutVersion() throws IOException { File testFile = new File(ROOT, "/invalidLayoutVersion"); File outputFile = new File(ROOT, "invalidLayoutVersionOutput"); @@ -190,7 +202,7 @@ public void unsupportedFSLayoutVersion(File originalFsimage) throws IOException } catch(IOException e) { if(!e.getMessage().contains(Integer.toString(badVersionNum))) throw e; // wasn't error we were expecting - System.out.println("Correctly failed at reading bad image version."); + LOG.debug("Correctly failed at reading bad image version."); } } finally { if(testFile.exists()) testFile.delete(); @@ -199,7 +211,8 @@ public void unsupportedFSLayoutVersion(File originalFsimage) throws IOException } // Verify that image viewer will bail on a file that ends unexpectedly - private void truncatedFSImage(File originalFsimage) throws IOException { + @Test + public void truncatedFSImage() throws IOException { File testFile = new File(ROOT, "/truncatedFSImage"); File outputFile = new File(ROOT, "/trucnatedFSImageOutput"); try { @@ -213,7 +226,7 @@ private void truncatedFSImage(File originalFsimage) throws IOException { oiv.go(); fail("Managed to process a truncated fsimage file"); } catch (EOFException e) { - System.out.println("Correctly handled EOF"); + LOG.debug("Correctly handled EOF"); } } finally { @@ -365,7 +378,8 @@ private void copyFile(File src, File dest) throws IOException { } } - private void outputOfFileDistributionVisitor(File originalFsimage) throws IOException { + @Test + public void outputOfFileDistributionVisitor() throws IOException { File testFile = new File(ROOT, "/basicCheck"); File outputFile = new File(ROOT, "/fileDistributionCheckOutput"); @@ -392,4 +406,66 @@ private void outputOfFileDistributionVisitor(File originalFsimage) throws IOExce } assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR); } + + private static class TestImageVisitor extends ImageVisitor { + private List delegationTokenRenewers = new LinkedList(); + TestImageVisitor() { + } + + List getDelegationTokenRenewers() { + return delegationTokenRenewers; + } + + @Override + void start() throws IOException { + } + + @Override + void finish() throws IOException { + } + + @Override + void finishAbnormally() throws IOException { + } + + @Override + void visit(ImageElement element, String value) throws IOException { + if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) { + delegationTokenRenewers.add(value); + } + } + + @Override + void visitEnclosingElement(ImageElement element) throws IOException { + } + + @Override + void visitEnclosingElement(ImageElement element, ImageElement key, + String value) throws IOException { + } + + @Override + void leaveEnclosingElement() throws IOException { + } + } + + @Test + public void outputOfTestVisitor() throws IOException { + File testFile = new File(ROOT, "/basicCheck"); + + try { + copyFile(originalFsimage, testFile); + TestImageVisitor v = new TestImageVisitor(); + OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true); + oiv.go(); + + // Validated stored delegation token identifiers. + List dtrs = v.getDelegationTokenRenewers(); + assertEquals(1, dtrs.size()); + assertEquals(TEST_RENEWER, dtrs.get(0)); + } finally { + if(testFile.exists()) testFile.delete(); + } + LOG.debug("Passed TestVisitor validation."); + } }