diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt index a0a86b8611..e9c34dc215 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt @@ -159,3 +159,5 @@ Branch-2802 Snapshot (Unreleased) HDFS-4500. Refactor snapshot INode methods. (szetszwo) HDFS-4487. Fix snapshot diff report for HDFS-4446. (Jing Zhao via szetszwo) + + HDFS-4431. Support snapshot in OfflineImageViewer. (Jing Zhao via szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java index 1c97dc6479..7e24e5cef0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java @@ -21,7 +21,6 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import org.apache.hadoop.hdfs.DFSUtil; @@ -199,22 +198,6 @@ private static List loadDeletedList(INodeDirectoryWithSnapshot parent, // useful, but set the parent here to be consistent with the original // fsdir tree. deleted.setParent(parent); - if (deleted instanceof INodeFile - && ((INodeFile) deleted).getBlocks() == null) { - // if deleted is an INodeFile, and its blocks is null, then deleted - // must be an INodeFileWithLink, and we need to rebuild its next link - int c = Collections.binarySearch(createdList, deleted.getLocalNameBytes()); - if (c < 0) { - throw new IOException( - "Cannot find the INode linked with the INode " - + deleted.getLocalName() - + " in deleted list while loading FSImage."); - } - // deleted must be an FileWithSnapshot (INodeFileSnapshot or - // INodeFileUnderConstructionSnapshot) - INodeFile cNode = (INodeFile) createdList.get(c); - ((INodeFile) deleted).setBlocks(cNode.getBlocks()); - } } return deletedList; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java index b4ee16b508..c1d9c89c67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java @@ -123,7 +123,7 @@ class ImageLoaderCurrent implements ImageLoader { new SimpleDateFormat("yyyy-MM-dd HH:mm"); private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38, -39, - -40}; + -40, -41, -42}; private int imageVersion = 0; /* (non-Javadoc) @@ -162,6 +162,14 @@ public void loadImage(DataInputStream in, ImageVisitor v, if (LayoutVersion.supports(Feature.STORED_TXIDS, imageVersion)) { v.visit(ImageElement.TRANSACTION_ID, in.readLong()); } + + boolean supportSnapshot = LayoutVersion.supports(Feature.SNAPSHOT, + imageVersion); + if (supportSnapshot) { + v.visit(ImageElement.SNAPSHOT_COUNTER, in.readInt()); + v.visit(ImageElement.NUM_SNAPSHOTS_TOTAL, in.readInt()); + v.visit(ImageElement.NUM_SNAPSHOTTABLE_DIRS, in.readInt()); + } if (LayoutVersion.supports(Feature.FSIMAGE_COMPRESSION, imageVersion)) { boolean isCompressed = in.readBoolean(); @@ -179,7 +187,7 @@ public void loadImage(DataInputStream in, ImageVisitor v, in = new DataInputStream(codec.createInputStream(in)); } } - processINodes(in, v, numInodes, skipBlocks); + processINodes(in, v, numInodes, skipBlocks, supportSnapshot); processINodesUC(in, v, skipBlocks); @@ -356,16 +364,22 @@ private void processPermission(DataInputStream in, ImageVisitor v) * @param v Visitor to walk over INodes * @param numInodes Number of INodes stored in file * @param skipBlocks Process all the blocks within the INode? + * @param supportSnapshot Whether or not the imageVersion supports snapshot * @throws VisitException * @throws IOException */ private void processINodes(DataInputStream in, ImageVisitor v, - long numInodes, boolean skipBlocks) throws IOException { + long numInodes, boolean skipBlocks, boolean supportSnapshot) + throws IOException { v.visitEnclosingElement(ImageElement.INODES, ImageElement.NUM_INODES, numInodes); if (LayoutVersion.supports(Feature.FSIMAGE_NAME_OPTIMIZATION, imageVersion)) { - processLocalNameINodes(in, v, numInodes, skipBlocks); + if (!supportSnapshot) { + processLocalNameINodes(in, v, numInodes, skipBlocks); + } else { + processLocalNameINodesWithSnapshot(in, v, skipBlocks); + } } else { // full path name processFullNameINodes(in, v, numInodes, skipBlocks); } @@ -386,7 +400,7 @@ private void processINodes(DataInputStream in, ImageVisitor v, private void processLocalNameINodes(DataInputStream in, ImageVisitor v, long numInodes, boolean skipBlocks) throws IOException { // process root - processINode(in, v, skipBlocks, ""); + processINode(in, v, skipBlocks, "", false); numInodes--; while (numInodes > 0) { numInodes -= processDirectory(in, v, skipBlocks); @@ -396,40 +410,163 @@ private void processLocalNameINodes(DataInputStream in, ImageVisitor v, private int processDirectory(DataInputStream in, ImageVisitor v, boolean skipBlocks) throws IOException { String parentName = FSImageSerialization.readString(in); + return processChildren(in, v, skipBlocks, parentName); + } + + /** + * Process image with local path name and snapshot support + * + * @param in image stream + * @param v visitor + * @param skipBlocks skip blocks or not + */ + private void processLocalNameINodesWithSnapshot(DataInputStream in, + ImageVisitor v, boolean skipBlocks) throws IOException { + // process root + processINode(in, v, skipBlocks, "", false); + processDirectoryWithSnapshot(in, v, skipBlocks); + } + + /** + * Process directories when snapshot is supported. + */ + private void processDirectoryWithSnapshot(DataInputStream in, ImageVisitor v, + boolean skipBlocks) throws IOException { + // 1. load dir name + String dirName = FSImageSerialization.readString(in); + // 2. load possible snapshots + processSnapshots(in, v, dirName); + // 3. load children nodes + processChildren(in, v, skipBlocks, dirName); + // 4. load possible directory diff list + processDirectoryDiffList(in, v, dirName); + // recursively process sub-directories + final int numSubTree = in.readInt(); + for (int i = 0; i < numSubTree; i++) { + processDirectoryWithSnapshot(in, v, skipBlocks); + } + } + + /** + * Process snapshots of a snapshottable directory + */ + private void processSnapshots(DataInputStream in, ImageVisitor v, + String rootName) throws IOException { + final int numSnapshots = in.readInt(); + if (numSnapshots >= 0) { + v.visitEnclosingElement(ImageElement.SNAPSHOTS, + ImageElement.NUM_SNAPSHOTS, numSnapshots); + for (int i = 0; i < numSnapshots; i++) { + // process snapshot + v.visitEnclosingElement(ImageElement.SNAPSHOT); + v.visit(ImageElement.SNAPSHOT_ID, in.readInt()); + // process root of snapshot + v.visitEnclosingElement(ImageElement.SNAPSHOT_ROOT); + processINode(in, v, true, rootName, false); + v.leaveEnclosingElement(); + v.leaveEnclosingElement(); + } + v.visit(ImageElement.SNAPSHOT_QUOTA, in.readInt()); + v.leaveEnclosingElement(); + } + } + + private void processDirectoryDiffList(DataInputStream in, ImageVisitor v, + String currentINodeName) throws IOException { + final int numDirDiff = in.readInt(); + if (numDirDiff >= 0) { + v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFFS, + ImageElement.NUM_SNAPSHOT_DIR_DIFF, numDirDiff); + for (int i = 0; i < numDirDiff; i++) { + // process directory diffs in reverse chronological oder + processDirectoryDiff(in, v, currentINodeName); + } + v.leaveEnclosingElement(); + } + } + + private void processDirectoryDiff(DataInputStream in, ImageVisitor v, + String currentINodeName) throws IOException { + v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFF); + String snapshot = FSImageSerialization.readString(in); + v.visit(ImageElement.SNAPSHOT_DIFF_SNAPSHOTROOT, snapshot); + v.visit(ImageElement.SNAPSHOT_DIR_DIFF_CHILDREN_SIZE, in.readInt()); + + // process snapshotINode + boolean useRoot = in.readBoolean(); + if (!useRoot) { + if (in.readBoolean()) { + v.visitEnclosingElement(ImageElement.SNAPSHOT_DIFF_SNAPSHOTINODE); + processINode(in, v, true, currentINodeName, true); + v.leaveEnclosingElement(); + } + } + + // process createdList + int createdSize = in.readInt(); + v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFF_CREATEDLIST, + ImageElement.SNAPSHOT_DIR_DIFF_CREATEDLIST_SIZE, createdSize); + for (int i = 0; i < createdSize; i++) { + String createdNode = FSImageSerialization.readString(in); + v.visit(ImageElement.SNAPSHOT_DIR_DIFF_CREATED_INODE, createdNode); + } + v.leaveEnclosingElement(); + + // process deletedList + int deletedSize = in.readInt(); + v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFF_DELETEDLIST, + ImageElement.SNAPSHOT_DIR_DIFF_DELETEDLIST_SIZE, deletedSize); + for (int i = 0; i < deletedSize; i++) { + v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFF_DELETED_INODE); + processINode(in, v, false, currentINodeName, true); + v.leaveEnclosingElement(); + } + v.leaveEnclosingElement(); + v.leaveEnclosingElement(); + } + + /** Process children under a directory */ + private int processChildren(DataInputStream in, ImageVisitor v, + boolean skipBlocks, String parentName) throws IOException { int numChildren = in.readInt(); - for (int i=0; i 0 || numBlocks == -1) { + + if (numBlocks > 0) { // File + if (supportSnapshot) { + // process file diffs + processFileDiffList(in, v, parentName); + if (isSnapshotCopy) { + boolean underConstruction = in.readBoolean(); + if (underConstruction) { + v.visit(ImageElement.CLIENT_NAME, + FSImageSerialization.readString(in)); + v.visit(ImageElement.CLIENT_MACHINE, + FSImageSerialization.readString(in)); + } + } + } + } else if (numBlocks == -1) { // Directory v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1); if (LayoutVersion.supports(Feature.DISKSPACE_QUOTA, imageVersion)) v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1); - } - if (numBlocks == -2) { + if (supportSnapshot) { + boolean snapshottable = in.readBoolean(); + if (!snapshottable) { + boolean withSnapshot = in.readBoolean(); + v.visit(ImageElement.IS_WITHSNAPSHOT_DIR, Boolean.toString(withSnapshot)); + } else { + v.visit(ImageElement.IS_SNAPSHOTTABLE_DIR, Boolean.toString(snapshottable)); + } + } + } else if (numBlocks == -2) { v.visit(ImageElement.SYMLINK, Text.readString(in)); } processPermission(in, v); v.leaveEnclosingElement(); // INode } - + + private void processFileDiffList(DataInputStream in, ImageVisitor v, + String currentINodeName) throws IOException { + final int size = in.readInt(); + if (size >= 0) { + v.visitEnclosingElement(ImageElement.SNAPSHOT_FILE_DIFFS, + ImageElement.NUM_SNAPSHOT_FILE_DIFF, size); + String snapshot = FSImageSerialization.readString(in); + v.visit(ImageElement.SNAPSHOT_DIFF_SNAPSHOTROOT, snapshot); + v.visit(ImageElement.SNAPSHOT_FILE_SIZE, in.readLong()); + if (in.readBoolean()) { + v.visitEnclosingElement(ImageElement.SNAPSHOT_DIFF_SNAPSHOTINODE); + processINode(in, v, true, currentINodeName, true); + v.leaveEnclosingElement(); + } + v.leaveEnclosingElement(); + } + } + /** * Helper method to format dates during processing. * @param date Date as read from image file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java index e1b2fda557..98dd04ab26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java @@ -80,7 +80,34 @@ public enum ImageElement { DELEGATION_TOKEN_IDENTIFIER_MAX_DATE, DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME, DELEGATION_TOKEN_IDENTIFIER_MASTER_KEY_ID, - TRANSACTION_ID + TRANSACTION_ID, + SNAPSHOT_COUNTER, + NUM_SNAPSHOTS_TOTAL, + NUM_SNAPSHOTTABLE_DIRS, + NUM_SNAPSHOTS, + SNAPSHOTS, + SNAPSHOT, + SNAPSHOT_ID, + SNAPSHOT_ROOT, + SNAPSHOT_QUOTA, + NUM_SNAPSHOT_DIR_DIFF, + SNAPSHOT_DIR_DIFFS, + SNAPSHOT_DIR_DIFF, + SNAPSHOT_DIFF_SNAPSHOTROOT, + SNAPSHOT_DIR_DIFF_CHILDREN_SIZE, + SNAPSHOT_DIFF_SNAPSHOTINODE, + SNAPSHOT_DIR_DIFF_CREATEDLIST, + SNAPSHOT_DIR_DIFF_CREATEDLIST_SIZE, + SNAPSHOT_DIR_DIFF_CREATED_INODE, + SNAPSHOT_DIR_DIFF_DELETEDLIST, + SNAPSHOT_DIR_DIFF_DELETEDLIST_SIZE, + SNAPSHOT_DIR_DIFF_DELETED_INODE, + IS_SNAPSHOTTABLE_DIR, + IS_WITHSNAPSHOT_DIR, + SNAPSHOT_FILE_DIFFS, + SNAPSHOT_FILE_DIFF, + NUM_SNAPSHOT_FILE_DIFF, + SNAPSHOT_FILE_SIZE } /**