big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-3486. offlineimageviewer can't read fsimage files that contain persistent delegation tokens. Contributed by Colin Patrick McCabe

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1344970 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-06-01 02:31:29 +00:00
parent 675a7e4acb
commit 4eb6d65eef
5 changed files with 166 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -273,6 +273,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3484. hdfs fsck doesn't work if NN HTTP address is set to
0.0.0.0 even if NN RPC address is configured. (atm via eli)
HDFS-3486. offlineimageviewer can't read fsimage files that contain
persistent delegation tokens. (Colin Patrick McCabe via eli)
Release 2.0.0-alpha - UNRELEASED
INCOMPATIBLE CHANGES

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
View File

@ -31,11 +31,13 @@
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor.ImageElement;
import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.xml.sax.helpers.AttributesImpl;
/**
* ImageLoaderCurrent processes Hadoop FSImage files and walks over
@ -220,9 +222,29 @@ private void processDelegationTokens(DataInputStream in, ImageVisitor v)
for(int i=0; i<numDTokens; i++){
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.readFields(in);
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER, id.toString());
long expiryTime = in.readLong();
v.visitEnclosingElement(ImageElement.DELEGATION_TOKEN_IDENTIFIER);
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_KIND,
id.getKind().toString());
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_SEQNO,
id.getSequenceNumber());
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_OWNER,
id.getOwner().toString());
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER,
id.getRenewer().toString());
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_REALUSER,
id.getRealUser().toString());
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE,
id.getIssueDate());
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_MAX_DATE,
id.getMaxDate());
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME,
expiryTime);
v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_MASTER_KEY_ID,
id.getMasterKeyId());
v.leaveEnclosingElement(); // DELEGATION_TOKEN_IDENTIFIER
}
v.leaveEnclosingElement();
v.leaveEnclosingElement(); // DELEGATION_TOKENS
}
/**

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
View File

@ -71,7 +71,15 @@ public enum ImageElement {
NUM_DELEGATION_TOKENS,
DELEGATION_TOKENS,
DELEGATION_TOKEN_IDENTIFIER,
DELEGATION_TOKEN_EXPIRY_TIME,
DELEGATION_TOKEN_IDENTIFIER_KIND,
DELEGATION_TOKEN_IDENTIFIER_SEQNO,
DELEGATION_TOKEN_IDENTIFIER_OWNER,
DELEGATION_TOKEN_IDENTIFIER_RENEWER,
DELEGATION_TOKEN_IDENTIFIER_REALUSER,
DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE,
DELEGATION_TOKEN_IDENTIFIER_MAX_DATE,
DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME,
DELEGATION_TOKEN_IDENTIFIER_MASTER_KEY_ID,
TRANSACTION_ID
}

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import java.io.IOException;
import java.util.Date;
/**
* IndentedImageVisitor walks over an FSImage and displays its structure
@ -58,6 +59,16 @@ void visit(ImageElement element, String value) throws IOException {
write(element + " = " + value + "\n");
}
void visit(ImageElement element, long value) throws IOException {
if ((element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME) ||
(element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE) ||
(element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_MAX_DATE)) {
visit(element, new Date(value).toString());
} else {
visit(element, Long.toString(value));
}
}
@Override
void visitEnclosingElement(ImageElement element) throws IOException {
printIndents();

160
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
View File

@ -18,6 +18,10 @@
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import java.io.BufferedReader;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.token.Token;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
@ -29,15 +33,19 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import junit.framework.TestCase;
import org.junit.*;
import static org.junit.Assert.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
@ -52,12 +60,15 @@
* * confirm it correctly bails on malformed image files, in particular, a
* file that ends suddenly.
*/
public class TestOfflineImageViewer extends TestCase {
public class TestOfflineImageViewer {
private static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
private static final int NUM_DIRS = 3;
private static final int FILES_PER_DIR = 4;
private static final String TEST_RENEWER = "JobTracker";
private static File originalFsimage = null;
// Elements of lines of ls-file output to be compared to FileStatus instance
private class LsElements {
private static class LsElements {
public String perms;
public int replication;
public String username;
@ -67,43 +78,28 @@ private class LsElements {
}
// namespace as written to dfs, to be compared with viewer's output
final HashMap<String, FileStatus> writtenFiles
= new HashMap<String, FileStatus>();
final static HashMap<String, FileStatus> writtenFiles =
new HashMap<String, FileStatus>();
private static String ROOT = System.getProperty("test.build.data",
"build/test/data");
// Main entry point into testing. Necessary since we only want to generate
// the fsimage file once and use it for multiple tests.
public void testOIV() throws Exception {
File originalFsimage = null;
try {
originalFsimage = initFsimage();
assertNotNull("originalFsImage shouldn't be null", originalFsimage);
// Tests:
outputOfLSVisitor(originalFsimage);
outputOfFileDistributionVisitor(originalFsimage);
unsupportedFSLayoutVersion(originalFsimage);
truncatedFSImage(originalFsimage);
} finally {
if(originalFsimage != null && originalFsimage.exists())
originalFsimage.delete();
}
}
// Create a populated namespace for later testing. Save its contents to a
// data structure and store its fsimage location.
private File initFsimage() throws IOException {
// We only want to generate the fsimage file once and use it for
// multiple tests.
@BeforeClass
public static void createOriginalFSImage() throws IOException {
MiniDFSCluster cluster = null;
File orig = null;
try {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
conf.set("hadoop.security.auth_to_local",
"RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
cluster.waitActive();
FileSystem hdfs = cluster.getFileSystem();
int filesize = 256;
@ -123,34 +119,49 @@ private File initFsimage() throws IOException {
}
}
// Get delegation tokens so we log the delegation token op
List<Token<?>> delegationTokens =
hdfs.getDelegationTokens(TEST_RENEWER);
for (Token<?> t : delegationTokens) {
LOG.debug("got token " + t);
}
// Write results to the fsimage file
cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER);
cluster.getNameNodeRpc().saveNamespace();
// Determine location of fsimage file
orig = FSImageTestUtil.findLatestImageFile(
originalFsimage = FSImageTestUtil.findLatestImageFile(
FSImageTestUtil.getFSImage(
cluster.getNameNode()).getStorage().getStorageDir(0));
if (orig == null) {
fail("Didn't generate or can't find fsimage");
if (originalFsimage == null) {
throw new RuntimeException("Didn't generate or can't find fsimage");
}
LOG.debug("original FS image file is " + originalFsimage);
} finally {
if(cluster != null)
cluster.shutdown();
}
return orig;
}
@AfterClass
public static void deleteOriginalFSImage() throws IOException {
if(originalFsimage != null && originalFsimage.exists()) {
originalFsimage.delete();
}
}
// Convenience method to generate a file status from file system for
// later comparison
private FileStatus pathToFileEntry(FileSystem hdfs, String file)
private static FileStatus pathToFileEntry(FileSystem hdfs, String file)
throws IOException {
return hdfs.getFileStatus(new Path(file));
}
// Verify that we can correctly generate an ls-style output for a valid
// fsimage
private void outputOfLSVisitor(File originalFsimage) throws IOException {
@Test
public void outputOfLSVisitor() throws IOException {
File testFile = new File(ROOT, "/basicCheck");
File outputFile = new File(ROOT, "/basicCheckOutput");
@ -169,12 +180,13 @@ private void outputOfLSVisitor(File originalFsimage) throws IOException {
if(testFile.exists()) testFile.delete();
if(outputFile.exists()) outputFile.delete();
}
System.out.println("Correctly generated ls-style output.");
LOG.debug("Correctly generated ls-style output.");
}
// Confirm that attempting to read an fsimage file with an unsupported
// layout results in an error
public void unsupportedFSLayoutVersion(File originalFsimage) throws IOException {
@Test
public void unsupportedFSLayoutVersion() throws IOException {
File testFile = new File(ROOT, "/invalidLayoutVersion");
File outputFile = new File(ROOT, "invalidLayoutVersionOutput");
@ -190,7 +202,7 @@ public void unsupportedFSLayoutVersion(File originalFsimage) throws IOException
} catch(IOException e) {
if(!e.getMessage().contains(Integer.toString(badVersionNum)))
throw e; // wasn't error we were expecting
System.out.println("Correctly failed at reading bad image version.");
LOG.debug("Correctly failed at reading bad image version.");
}
} finally {
if(testFile.exists()) testFile.delete();
@ -199,7 +211,8 @@ public void unsupportedFSLayoutVersion(File originalFsimage) throws IOException
}
// Verify that image viewer will bail on a file that ends unexpectedly
private void truncatedFSImage(File originalFsimage) throws IOException {
@Test
public void truncatedFSImage() throws IOException {
File testFile = new File(ROOT, "/truncatedFSImage");
File outputFile = new File(ROOT, "/trucnatedFSImageOutput");
try {
@ -213,7 +226,7 @@ private void truncatedFSImage(File originalFsimage) throws IOException {
oiv.go();
fail("Managed to process a truncated fsimage file");
} catch (EOFException e) {
System.out.println("Correctly handled EOF");
LOG.debug("Correctly handled EOF");
}
} finally {
@ -365,7 +378,8 @@ private void copyFile(File src, File dest) throws IOException {
}
}
private void outputOfFileDistributionVisitor(File originalFsimage) throws IOException {
@Test
public void outputOfFileDistributionVisitor() throws IOException {
File testFile = new File(ROOT, "/basicCheck");
File outputFile = new File(ROOT, "/fileDistributionCheckOutput");
@ -392,4 +406,66 @@ private void outputOfFileDistributionVisitor(File originalFsimage) throws IOExce
}
assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR);
}
private static class TestImageVisitor extends ImageVisitor {
private List<String> delegationTokenRenewers = new LinkedList<String>();
TestImageVisitor() {
}
List<String> getDelegationTokenRenewers() {
return delegationTokenRenewers;
}
@Override
void start() throws IOException {
}
@Override
void finish() throws IOException {
}
@Override
void finishAbnormally() throws IOException {
}
@Override
void visit(ImageElement element, String value) throws IOException {
if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) {
delegationTokenRenewers.add(value);
}
}
@Override
void visitEnclosingElement(ImageElement element) throws IOException {
}
@Override
void visitEnclosingElement(ImageElement element, ImageElement key,
String value) throws IOException {
}
@Override
void leaveEnclosingElement() throws IOException {
}
}
@Test
public void outputOfTestVisitor() throws IOException {
File testFile = new File(ROOT, "/basicCheck");
try {
copyFile(originalFsimage, testFile);
TestImageVisitor v = new TestImageVisitor();
OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true);
oiv.go();
// Validated stored delegation token identifiers.
List<String> dtrs = v.getDelegationTokenRenewers();
assertEquals(1, dtrs.size());
assertEquals(TEST_RENEWER, dtrs.get(0));
} finally {
if(testFile.exists()) testFile.delete();
}
LOG.debug("Passed TestVisitor validation.");
}
}