From 8a83bb7ad6177f473c20c4cc9c0f46746224332c Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Sat, 21 Jun 2014 22:59:34 +0000 Subject: [PATCH] HDFS-4667. Capture renamed files/directories in snapshot diff report. Contributed by Jing Zhao and Binglin Chang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604488 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/protocol/SnapshotDiffReport.java | 58 +++- .../hadoop/hdfs/protocolPB/PBHelper.java | 28 +- .../hdfs/server/namenode/FSDirectory.java | 49 +-- .../hdfs/server/namenode/INodeDirectory.java | 8 +- .../namenode/INodeDirectoryAttributes.java | 6 +- .../hdfs/server/namenode/INodeFile.java | 9 + .../server/namenode/INodeFileAttributes.java | 11 + .../hdfs/server/namenode/INodeReference.java | 31 +- .../snapshot/AbstractINodeDiffList.java | 40 +-- .../DirectoryWithSnapshotFeature.java | 83 ++--- .../snapshot/FileWithSnapshotFeature.java | 37 ++- .../snapshot/INodeDirectorySnapshottable.java | 157 ++++++++-- .../hadoop-hdfs/src/main/proto/hdfs.proto | 1 + .../src/site/xdoc/HdfsSnapshots.xml | 28 ++ .../TestFullPathNameWithSnapshot.java | 295 ------------------ .../snapshot/TestRenameWithSnapshots.java | 75 ++--- .../snapshot/TestSnapshotDiffReport.java | 170 +++++++++- 18 files changed, 560 insertions(+), 529 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5cbea0dd7c..2ffbee85d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -458,6 +458,9 @@ Release 2.5.0 - UNRELEASED HDFS-6557. Move the reference of fsimage to FSNamesystem. (wheat9) + HDFS-4667. Capture renamed files/directories in snapshot diff report. (jing9 + and Binglin Chang via jing9) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index 265a05d08e..ba96371ffa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -25,6 +25,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo; +import com.google.common.base.Objects; + /** * This class represents to end users the difference between two snapshots of * the same directory, or the difference between a snapshot of the directory and @@ -79,43 +81,64 @@ public class SnapshotDiffReport { /** The type of the difference. */ private final DiffType type; /** - * The relative path (related to the snapshot root) of the file/directory - * where changes have happened + * The relative path (related to the snapshot root) of 1) the file/directory + * where changes have happened, or 2) the source file/dir of a rename op. */ - private final byte[] relativePath; + private final byte[] sourcePath; + private final byte[] targetPath; - public DiffReportEntry(DiffType type, byte[] path) { + public DiffReportEntry(DiffType type, byte[] sourcePath) { + this(type, sourcePath, null); + } + + public DiffReportEntry(DiffType type, byte[][] sourcePathComponents) { + this(type, sourcePathComponents, null); + } + + public DiffReportEntry(DiffType type, byte[] sourcePath, byte[] targetPath) { this.type = type; - this.relativePath = path; + this.sourcePath = sourcePath; + this.targetPath = targetPath; } - public DiffReportEntry(DiffType type, byte[][] pathComponents) { + public DiffReportEntry(DiffType type, byte[][] sourcePathComponents, + byte[][] targetPathComponents) { this.type = type; - this.relativePath = DFSUtil.byteArray2bytes(pathComponents); + this.sourcePath = DFSUtil.byteArray2bytes(sourcePathComponents); + this.targetPath = targetPathComponents == null ? null : DFSUtil + .byteArray2bytes(targetPathComponents); } @Override public String toString() { - return type.getLabel() + "\t" + getRelativePathString(); + String str = type.getLabel() + "\t" + getPathString(sourcePath); + if (type == DiffType.RENAME) { + str += " -> " + getPathString(targetPath); + } + return str; } public DiffType getType() { return type; } - public String getRelativePathString() { - String path = DFSUtil.bytes2String(relativePath); - if (path.isEmpty()) { + static String getPathString(byte[] path) { + String pathStr = DFSUtil.bytes2String(path); + if (pathStr.isEmpty()) { return Path.CUR_DIR; } else { - return Path.CUR_DIR + Path.SEPARATOR + path; + return Path.CUR_DIR + Path.SEPARATOR + pathStr; } } - public byte[] getRelativePath() { - return relativePath; + public byte[] getSourcePath() { + return sourcePath; } - + + public byte[] getTargetPath() { + return targetPath; + } + @Override public boolean equals(Object other) { if (this == other) { @@ -124,14 +147,15 @@ public class SnapshotDiffReport { if (other != null && other instanceof DiffReportEntry) { DiffReportEntry entry = (DiffReportEntry) other; return type.equals(entry.getType()) - && Arrays.equals(relativePath, entry.getRelativePath()); + && Arrays.equals(sourcePath, entry.getSourcePath()) + && Arrays.equals(targetPath, entry.getTargetPath()); } return false; } @Override public int hashCode() { - return Arrays.hashCode(relativePath); + return Objects.hashCode(getSourcePath(), getTargetPath()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index f3b62c07f1..e9435024be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -44,7 +44,6 @@ import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.proto.HAServiceProtocolProtos; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.StorageType; -import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; @@ -1737,24 +1736,29 @@ public class PBHelper { } DiffType type = DiffType.getTypeFromLabel(entry .getModificationLabel()); - return type == null ? null : - new DiffReportEntry(type, entry.getFullpath().toByteArray()); + return type == null ? null : new DiffReportEntry(type, entry.getFullpath() + .toByteArray(), entry.hasTargetPath() ? entry.getTargetPath() + .toByteArray() : null); } public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { if (entry == null) { return null; } - byte[] fullPath = entry.getRelativePath(); - ByteString fullPathString = ByteString - .copyFrom(fullPath == null ? DFSUtil.EMPTY_BYTES : fullPath); - + ByteString sourcePath = ByteString + .copyFrom(entry.getSourcePath() == null ? DFSUtil.EMPTY_BYTES : entry + .getSourcePath()); String modification = entry.getType().getLabel(); - - SnapshotDiffReportEntryProto entryProto = SnapshotDiffReportEntryProto - .newBuilder().setFullpath(fullPathString) - .setModificationLabel(modification).build(); - return entryProto; + SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto + .newBuilder().setFullpath(sourcePath) + .setModificationLabel(modification); + if (entry.getType() == DiffType.RENAME) { + ByteString targetPath = ByteString + .copyFrom(entry.getTargetPath() == null ? DFSUtil.EMPTY_BYTES : entry + .getTargetPath()); + builder.setTargetPath(targetPath); + } + return builder.build(); } public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index d00372957f..d9e3be931d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -2790,7 +2790,7 @@ public class FSDirectory implements Closeable { * Note that this method cannot handle scenarios where the inode is in a * snapshot. */ - static byte[][] getPathComponents(INode inode) { + public static byte[][] getPathComponents(INode inode) { List components = new ArrayList(); components.add(0, inode.getLocalNameBytes()); while(inode.getParent() != null) { @@ -2800,53 +2800,6 @@ public class FSDirectory implements Closeable { return components.toArray(new byte[components.size()][]); } - /** - * The same functionality with {@link #getPathComponents(INode)}, but can - * handle snapshots. - */ - public static byte[][] getPathComponentsWithSnapshot(INode inode) { - List components = new ArrayList(); - boolean inSnapshot = false; - int snapshotId = Snapshot.CURRENT_STATE_ID; - do { - if (inode instanceof INodeReference.WithCount) { - // identify the corresponding WithName or DstReference node - inode = ((WithCount) inode).getParentRef(snapshotId); - } else { // normal INode and WithName/DstReference - if (inode instanceof INodeDirectory - && inode.asDirectory().isSnapshottable() && inSnapshot - && snapshotId != Snapshot.CURRENT_STATE_ID) { - INodeDirectorySnapshottable sdir = (INodeDirectorySnapshottable) inode - .asDirectory(); - Snapshot snapshot = sdir.getSnapshotById(snapshotId); - if (snapshot != null) { - components.add(0, snapshot.getRoot().getLocalNameBytes()); - components.add(0, HdfsConstants.DOT_SNAPSHOT_DIR_BYTES); - // the snapshot has been found, thus no need to check snapshottable - // directory afterwards - inSnapshot = false; - } - } - INode parent = inode.getParentReference() != null ? inode - .getParentReference() : inode.getParent(); - if (parent != null && parent instanceof INodeDirectory) { - int sid = parent.asDirectory().searchChild(inode); - Preconditions.checkState(sid != Snapshot.NO_SNAPSHOT_ID); - if (sid != Snapshot.CURRENT_STATE_ID - && snapshotId == Snapshot.CURRENT_STATE_ID) { - snapshotId = sid; - inSnapshot = true; - } - components.add(0, inode.getLocalNameBytes()); - } else if (parent == null) { // root - components.add(0, inode.getLocalNameBytes()); - } - inode = parent; - } - } while (inode != null); - return components.toArray(new byte[components.size()][]); - } - /** * @return path components for reserved path, else null. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index e5080a3c5c..8781934bee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -26,11 +26,9 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.fs.PathIsNotDirectoryException; -import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; -import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList; @@ -365,7 +363,7 @@ public class INodeDirectory extends INodeWithAdditionalFields * children list nor in any snapshot; otherwise the snapshot id of the * corresponding snapshot diff list. */ - int searchChild(INode inode) { + public int searchChild(INode inode) { INode child = getChild(inode.getLocalNameBytes(), Snapshot.CURRENT_STATE_ID); if (child != inode) { // inode is not in parent's children list, thus inode must be in @@ -764,7 +762,9 @@ public class INodeDirectory extends INodeWithAdditionalFields public boolean metadataEquals(INodeDirectoryAttributes other) { return other != null && getQuotaCounts().equals(other.getQuotaCounts()) - && getPermissionLong() == other.getPermissionLong(); + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); } /* diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java index d95fa4651b..b1e74859a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java @@ -53,8 +53,10 @@ public interface INodeDirectoryAttributes extends INodeAttributes { @Override public boolean metadataEquals(INodeDirectoryAttributes other) { return other != null - && this.getQuotaCounts().equals(other.getQuotaCounts()) - && getPermissionLong() == other.getPermissionLong(); + && getQuotaCounts().equals(other.getQuotaCounts()) + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index a38e8d9772..42025348af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -144,6 +144,15 @@ public class INodeFile extends INodeWithAdditionalFields return this; } + @Override + public boolean metadataEquals(INodeFileAttributes other) { + return other != null + && getHeaderLong()== other.getHeaderLong() + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); + } + /* Start of Under-Construction Feature */ /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java index 127642506a..37bf0884cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java @@ -36,6 +36,8 @@ public interface INodeFileAttributes extends INodeAttributes { /** @return the header as a long. */ public long getHeaderLong(); + public boolean metadataEquals(INodeFileAttributes other); + /** A copy of the inode file attributes */ public static class SnapshotCopy extends INodeAttributes.SnapshotCopy implements INodeFileAttributes { @@ -70,5 +72,14 @@ public interface INodeFileAttributes extends INodeAttributes { public long getHeaderLong() { return header; } + + @Override + public boolean metadataEquals(INodeFileAttributes other) { + return other != null + && getHeaderLong()== other.getHeaderLong() + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java index 461f075a0b..ac0f19d032 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java @@ -435,21 +435,28 @@ public abstract class INodeReference extends INode { } } + /** + * @return the WithName/DstReference node contained in the given snapshot. + */ public INodeReference getParentRef(int snapshotId) { - // when the given snapshotId is CURRENT_STATE_ID, it is possible that we - // do not know where the corresponding inode belongs, thus we simply - // return the last reference node - if (snapshotId == Snapshot.CURRENT_STATE_ID) { - return this.getParentReference() != null ? this.getParentReference() - : this.getLastWithName(); - } - // otherwise we search the withNameList - for (int i = 0; i < withNameList.size(); i++) { - if (snapshotId <= withNameList.get(i).lastSnapshotId) { - return withNameList.get(i); + int start = 0; + int end = withNameList.size() - 1; + while (start < end) { + int mid = start + (end - start) / 2; + int sid = withNameList.get(mid).lastSnapshotId; + if (sid == snapshotId) { + return withNameList.get(mid); + } else if (sid < snapshotId) { + start = mid + 1; + } else { + end = mid; } } - return this.getParentReference(); + if (withNameList.get(start).lastSnapshotId >= snapshotId) { + return withNameList.get(start); + } else { + return this.getParentReference(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java index 3c16b2ed52..d918495765 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java @@ -227,32 +227,34 @@ abstract class AbstractINodeDiffList 0) { + earlier = to; + later = from; + } + final int size = diffs.size(); int earlierDiffIndex = Collections.binarySearch(diffs, earlier.getId()); + int laterDiffIndex = later == null ? size : Collections + .binarySearch(diffs, later.getId()); if (-earlierDiffIndex - 1 == size) { // if the earlierSnapshot is after the latest SnapshotDiff stored in // diffs, no modification happened after the earlierSnapshot - return false; + return null; } - if (later != null) { - int laterDiffIndex = Collections.binarySearch(diffs, later.getId()); - if (laterDiffIndex == -1 || laterDiffIndex == 0) { - // if the laterSnapshot is the earliest SnapshotDiff stored in diffs, or - // before it, no modification happened before the laterSnapshot - return false; - } + if (laterDiffIndex == -1 || laterDiffIndex == 0) { + // if the laterSnapshot is the earliest SnapshotDiff stored in diffs, or + // before it, no modification happened before the laterSnapshot + return null; } - return true; + earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1) + : earlierDiffIndex; + laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1) + : laterDiffIndex; + return new int[]{earlierDiffIndex, laterDiffIndex}; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index c82309358e..e6645da4f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -21,7 +21,6 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.Iterator; @@ -42,6 +41,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.Quota; +import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo.RenameEntry; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; import org.apache.hadoop.hdfs.util.Diff; import org.apache.hadoop.hdfs.util.Diff.Container; @@ -170,49 +170,33 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { * @return A list of {@link DiffReportEntry} as the diff report. */ public List generateReport(byte[][] parentPath, - boolean fromEarlier) { - List cList = new ArrayList(); - List dList = new ArrayList(); - int c = 0, d = 0; + boolean fromEarlier, Map renameMap) { + List list = new ArrayList(); List created = getList(ListType.CREATED); List deleted = getList(ListType.DELETED); byte[][] fullPath = new byte[parentPath.length + 1][]; System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length); - for (; c < created.size() && d < deleted.size(); ) { - INode cnode = created.get(c); - INode dnode = deleted.get(d); - if (cnode.compareTo(dnode.getLocalNameBytes()) == 0) { + for (INode cnode : created) { + RenameEntry entry = renameMap.get(cnode.getId()); + if (entry == null || !entry.isRename()) { fullPath[fullPath.length - 1] = cnode.getLocalNameBytes(); - // must be the case: delete first and then create an inode with the - // same name - cList.add(new DiffReportEntry(DiffType.CREATE, fullPath)); - dList.add(new DiffReportEntry(DiffType.DELETE, fullPath)); - c++; - d++; - } else if (cnode.compareTo(dnode.getLocalNameBytes()) < 0) { - fullPath[fullPath.length - 1] = cnode.getLocalNameBytes(); - cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE + list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE : DiffType.DELETE, fullPath)); - c++; - } else { - fullPath[fullPath.length - 1] = dnode.getLocalNameBytes(); - dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE - : DiffType.CREATE, fullPath)); - d++; } } - for (; d < deleted.size(); d++) { - fullPath[fullPath.length - 1] = deleted.get(d).getLocalNameBytes(); - dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE - : DiffType.CREATE, fullPath)); + for (INode dnode : deleted) { + RenameEntry entry = renameMap.get(dnode.getId()); + if (entry != null && entry.isRename()) { + list.add(new DiffReportEntry(DiffType.RENAME, + fromEarlier ? entry.getSourcePath() : entry.getTargetPath(), + fromEarlier ? entry.getTargetPath() : entry.getSourcePath())); + } else { + fullPath[fullPath.length - 1] = dnode.getLocalNameBytes(); + list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE + : DiffType.CREATE, fullPath)); + } } - for (; c < created.size(); c++) { - fullPath[fullPath.length - 1] = created.get(c).getLocalNameBytes(); - cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE - : DiffType.DELETE, fullPath)); - } - dList.addAll(cList); - return dList; + return list; } } @@ -724,34 +708,21 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { */ boolean computeDiffBetweenSnapshots(Snapshot fromSnapshot, Snapshot toSnapshot, ChildrenDiff diff, INodeDirectory currentINode) { - Snapshot earlier = fromSnapshot; - Snapshot later = toSnapshot; - if (Snapshot.ID_COMPARATOR.compare(fromSnapshot, toSnapshot) > 0) { - earlier = toSnapshot; - later = fromSnapshot; - } - - boolean modified = diffs.changedBetweenSnapshots(earlier, later); - if (!modified) { + int[] diffIndexPair = diffs.changedBetweenSnapshots(fromSnapshot, + toSnapshot); + if (diffIndexPair == null) { return false; } - - final List difflist = diffs.asList(); - final int size = difflist.size(); - int earlierDiffIndex = Collections.binarySearch(difflist, earlier.getId()); - int laterDiffIndex = later == null ? size : Collections - .binarySearch(difflist, later.getId()); - earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1) - : earlierDiffIndex; - laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1) - : laterDiffIndex; + int earlierDiffIndex = diffIndexPair[0]; + int laterDiffIndex = diffIndexPair[1]; boolean dirMetadataChanged = false; INodeDirectoryAttributes dirCopy = null; + List difflist = diffs.asList(); for (int i = earlierDiffIndex; i < laterDiffIndex; i++) { DirectoryDiff sdiff = difflist.get(i); diff.combinePosterior(sdiff.diff, null); - if (dirMetadataChanged == false && sdiff.snapshotINode != null) { + if (!dirMetadataChanged && sdiff.snapshotINode != null) { if (dirCopy == null) { dirCopy = sdiff.snapshotINode; } else if (!dirCopy.metadataEquals(sdiff.snapshotINode)) { @@ -763,7 +734,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { if (!diff.isEmpty() || dirMetadataChanged) { return true; } else if (dirCopy != null) { - for (int i = laterDiffIndex; i < size; i++) { + for (int i = laterDiffIndex; i < difflist.size(); i++) { if (!dirCopy.metadataEquals(difflist.get(i).snapshotINode)) { return true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java index 52adfc6dd6..9e57b15195 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes; import org.apache.hadoop.hdfs.server.namenode.Quota; /** @@ -73,7 +74,41 @@ public class FileWithSnapshotFeature implements INode.Feature { } return max; } - + + boolean changedBetweenSnapshots(INodeFile file, Snapshot from, Snapshot to) { + int[] diffIndexPair = diffs.changedBetweenSnapshots(from, to); + if (diffIndexPair == null) { + return false; + } + int earlierDiffIndex = diffIndexPair[0]; + int laterDiffIndex = diffIndexPair[1]; + + final List diffList = diffs.asList(); + final long earlierLength = diffList.get(earlierDiffIndex).getFileSize(); + final long laterLength = laterDiffIndex == diffList.size() ? file + .computeFileSize(true, false) : diffList.get(laterDiffIndex) + .getFileSize(); + if (earlierLength != laterLength) { // file length has been changed + return true; + } + + INodeFileAttributes earlierAttr = null; // check the metadata + for (int i = earlierDiffIndex; i < laterDiffIndex; i++) { + FileDiff diff = diffList.get(i); + if (diff.snapshotINode != null) { + earlierAttr = diff.snapshotINode; + break; + } + } + if (earlierAttr == null) { // no meta-change at all, return false + return false; + } + INodeFileAttributes laterAttr = diffs.getSnapshotINode( + Math.max(Snapshot.getSnapshotId(from), Snapshot.getSnapshotId(to)), + file); + return !earlierAttr.metadataEquals(laterAttr); + } + public String getDetailedString() { return (isCurrentFileDeleted()? "(DELETED), ": ", ") + diffs; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java index 4288ec2afb..eef9215709 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.SortedMap; @@ -43,6 +44,9 @@ import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeMap; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; +import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; +import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName; import org.apache.hadoop.hdfs.server.namenode.Quota; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff; @@ -51,6 +55,7 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.util.Time; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.primitives.SignedBytes; /** @@ -98,7 +103,43 @@ public class INodeDirectorySnapshottable extends INodeDirectory { } } }; - + + static class RenameEntry { + private byte[][] sourcePath; + private byte[][] targetPath; + + void setSource(INode source, byte[][] sourceParentPath) { + Preconditions.checkState(sourcePath == null); + sourcePath = new byte[sourceParentPath.length + 1][]; + System.arraycopy(sourceParentPath, 0, sourcePath, 0, + sourceParentPath.length); + sourcePath[sourcePath.length - 1] = source.getLocalNameBytes(); + } + + void setTarget(INode target, byte[][] targetParentPath) { + targetPath = new byte[targetParentPath.length + 1][]; + System.arraycopy(targetParentPath, 0, targetPath, 0, + targetParentPath.length); + targetPath[targetPath.length - 1] = target.getLocalNameBytes(); + } + + void setTarget(byte[][] targetPath) { + this.targetPath = targetPath; + } + + boolean isRename() { + return sourcePath != null && targetPath != null; + } + + byte[][] getSourcePath() { + return sourcePath; + } + + byte[][] getTargetPath() { + return targetPath; + } + } + /** The root directory of the snapshots */ private final INodeDirectorySnapshottable snapshotRoot; /** The starting point of the difference */ @@ -109,7 +150,7 @@ public class INodeDirectorySnapshottable extends INodeDirectory { * A map recording modified INodeFile and INodeDirectory and their relative * path corresponding to the snapshot root. Sorted based on their names. */ - private final SortedMap diffMap = + private final SortedMap diffMap = new TreeMap(INODE_COMPARATOR); /** * A map capturing the detailed difference about file creation/deletion. @@ -119,7 +160,10 @@ public class INodeDirectorySnapshottable extends INodeDirectory { */ private final Map dirDiffMap = new HashMap(); - + + private final Map renameMap = + new HashMap(); + SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start, Snapshot end) { this.snapshotRoot = snapshotRoot; @@ -132,8 +176,36 @@ public class INodeDirectorySnapshottable extends INodeDirectory { ChildrenDiff diff) { dirDiffMap.put(dir, diff); diffMap.put(dir, relativePath); + // detect rename + for (INode created : diff.getList(ListType.CREATED)) { + if (created.isReference()) { + RenameEntry entry = getEntry(created.getId()); + if (entry.getTargetPath() == null) { + entry.setTarget(created, relativePath); + } + } + } + for (INode deleted : diff.getList(ListType.DELETED)) { + if (deleted instanceof INodeReference.WithName) { + RenameEntry entry = getEntry(deleted.getId()); + entry.setSource(deleted, relativePath); + } + } } - + + private RenameEntry getEntry(long inodeId) { + RenameEntry entry = renameMap.get(inodeId); + if (entry == null) { + entry = new RenameEntry(); + renameMap.put(inodeId, entry); + } + return entry; + } + + private void setRenameTarget(long inodeId, byte[][] path) { + getEntry(inodeId).setTarget(path); + } + /** Add a modified file */ private void addFileDiff(INodeFile file, byte[][] relativePath) { diffMap.put(file, relativePath); @@ -152,11 +224,11 @@ public class INodeDirectorySnapshottable extends INodeDirectory { List diffReportList = new ArrayList(); for (INode node : diffMap.keySet()) { diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap - .get(node))); + .get(node), null)); if (node.isDirectory()) { ChildrenDiff dirDiff = dirDiffMap.get(node); List subList = dirDiff.generateReport( - diffMap.get(node), isFromEarlier()); + diffMap.get(node), isFromEarlier(), renameMap); diffReportList.addAll(subList); } } @@ -423,25 +495,37 @@ public class INodeDirectorySnapshottable extends INodeDirectory { */ private void computeDiffRecursively(INode node, List parentPath, SnapshotDiffInfo diffReport) { - ChildrenDiff diff = new ChildrenDiff(); + final Snapshot earlierSnapshot = diffReport.isFromEarlier() ? + diffReport.from : diffReport.to; + final Snapshot laterSnapshot = diffReport.isFromEarlier() ? + diffReport.to : diffReport.from; byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]); if (node.isDirectory()) { + final ChildrenDiff diff = new ChildrenDiff(); INodeDirectory dir = node.asDirectory(); DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature(); if (sf != null) { - boolean change = sf.computeDiffBetweenSnapshots(diffReport.from, - diffReport.to, diff, dir); + boolean change = sf.computeDiffBetweenSnapshots(earlierSnapshot, + laterSnapshot, diff, dir); if (change) { diffReport.addDirDiff(dir, relativePath, diff); } } - ReadOnlyList children = dir.getChildrenList( - diffReport.isFromEarlier() ? Snapshot.getSnapshotId(diffReport.to) : - Snapshot.getSnapshotId(diffReport.from)); + ReadOnlyList children = dir.getChildrenList(earlierSnapshot + .getId()); for (INode child : children) { final byte[] name = child.getLocalNameBytes(); - if (diff.searchIndex(ListType.CREATED, name) < 0 - && diff.searchIndex(ListType.DELETED, name) < 0) { + boolean toProcess = diff.searchIndex(ListType.DELETED, name) < 0; + if (!toProcess && child instanceof INodeReference.WithName) { + byte[][] renameTargetPath = findRenameTargetPath((WithName) child, + laterSnapshot == null ? Snapshot.CURRENT_STATE_ID : + laterSnapshot.getId()); + if (renameTargetPath != null) { + toProcess = true; + diffReport.setRenameTarget(child.getId(), renameTargetPath); + } + } + if (toProcess) { parentPath.add(name); computeDiffRecursively(child, parentPath, diffReport); parentPath.remove(parentPath.size() - 1); @@ -449,18 +533,47 @@ public class INodeDirectorySnapshottable extends INodeDirectory { } } else if (node.isFile() && node.asFile().isWithSnapshot()) { INodeFile file = node.asFile(); - Snapshot earlierSnapshot = diffReport.isFromEarlier() ? diffReport.from - : diffReport.to; - Snapshot laterSnapshot = diffReport.isFromEarlier() ? diffReport.to - : diffReport.from; - boolean change = file.getDiffs().changedBetweenSnapshots(earlierSnapshot, - laterSnapshot); + boolean change = file.getFileWithSnapshotFeature() + .changedBetweenSnapshots(file, earlierSnapshot, laterSnapshot); if (change) { diffReport.addFileDiff(file, relativePath); } } } - + + /** + * We just found a deleted WithName node as the source of a rename operation. + * However, we should include it in our snapshot diff report as rename only + * if the rename target is also under the same snapshottable directory. + */ + private byte[][] findRenameTargetPath(INodeReference.WithName wn, + final int snapshotId) { + INode inode = wn.getReferredINode(); + final LinkedList ancestors = Lists.newLinkedList(); + while (inode != null) { + if (inode == this) { + return ancestors.toArray(new byte[ancestors.size()][]); + } + if (inode instanceof INodeReference.WithCount) { + inode = ((WithCount) inode).getParentRef(snapshotId); + } else { + INode parent = inode.getParentReference() != null ? inode + .getParentReference() : inode.getParent(); + if (parent != null && parent instanceof INodeDirectory) { + int sid = parent.asDirectory().searchChild(inode); + if (sid < snapshotId) { + return null; + } + } + if (!(parent instanceof WithCount)) { + ancestors.addFirst(inode.getLocalNameBytes()); + } + inode = parent; + } + } + return null; + } + /** * Replace itself with {@link INodeDirectoryWithSnapshot} or * {@link INodeDirectory} depending on the latest snapshot. @@ -549,4 +662,4 @@ public class INodeDirectorySnapshottable extends INodeDirectory { }); } } -} +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 2d7ca245a7..6fbb5b3420 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -275,6 +275,7 @@ message SnapshottableDirectoryListingProto { message SnapshotDiffReportEntryProto { required bytes fullpath = 1; required string modificationLabel = 2; + optional bytes targetPath = 3; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml index bd499c79c8..f809e855b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml @@ -255,7 +255,35 @@ fromSnapshotThe name of the starting snapshot. toSnapshotThe name of the ending snapshot. +
  • Results: + + + + + +
    +The file/directory has been created.
    -The file/directory has been deleted.
    MThe file/directory has been modified.
    RThe file/directory has been renamed.
    +
    • +

    + A RENAME entry indicates a file/directory has been renamed but + is still under the same snapshottable directory. A file/directory is + reported as deleted if it was renamed to outside of the snapshottble directory. + A file/directory renamed from outside of the snapshottble directory is + reported as newly created. +

    +

    + The snapshot difference report does not guarantee the same operation sequence. + For example, if we rename the directory "/foo" to "/foo2", and + then append new data to the file "/foo2/bar", the difference report will + be: + + R. /foo -> /foo2 + M. /foo/bar + + I.e., the changes on the files/directories under a renamed directory is + reported using the original path before the rename ("/foo/bar" in + the above example). +

    See also the corresponding Java API SnapshotDiffReport getSnapshotDiffReport(Path path, String fromSnapshot, String toSnapshot) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java deleted file mode 100644 index 23a6adf2bc..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode.snapshot; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.namenode.FSDirectory; -import org.apache.hadoop.hdfs.server.namenode.INode; -import org.apache.hadoop.hdfs.server.namenode.INodeReference; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - * Test getting the full path name of a given inode. The INode may be in - * snapshot. - */ -public class TestFullPathNameWithSnapshot { - private static final int BLOCKSIZE = 1024; - private static final short REPLICATION = 1; - - private Configuration conf; - private MiniDFSCluster cluster; - private FSDirectory fsdir; - private DistributedFileSystem dfs; - - @Before - public void setUp() throws Exception { - conf = new Configuration(); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) - .build(); - cluster.waitActive(); - - fsdir = cluster.getNamesystem().getFSDirectory(); - dfs = cluster.getFileSystem(); - } - - @After - public void tearDown() throws Exception { - if (cluster != null) { - cluster.shutdown(); - } - } - - /** - * Normal case without snapshot involved - */ - @Test - public void testNormalINode() throws Exception { - final Path bar = new Path("/foo/bar"); - dfs.mkdirs(bar); - final Path file = new Path(bar, "file"); - DFSTestUtil.createFile(dfs, file, BLOCKSIZE, REPLICATION, 0L); - INode fileNode = fsdir.getINode4Write(file.toString()); - byte[][] pathComponents = FSDirectory - .getPathComponentsWithSnapshot(fileNode); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(file.toString()), - pathComponents); - } - - /** - * INode in deleted list - */ - @Test - public void testDeletedINode() throws Exception { - final Path foo = new Path("/foo"); - final Path f1 = new Path(foo, "f1"); - DFSTestUtil.createFile(dfs, f1, BLOCKSIZE, REPLICATION, 0L); - final Path bar = new Path(foo, "bar"); - dfs.mkdirs(bar); - final Path f2 = new Path(bar, "f2"); - DFSTestUtil.createFile(dfs, f2, BLOCKSIZE, REPLICATION, 0L); - - INode f1Node = fsdir.getINode4Write(f1.toString()); - INode f2Node = fsdir.getINode4Write(f2.toString()); - - SnapshotTestHelper.createSnapshot(dfs, foo, "s1"); - dfs.delete(bar, true); - SnapshotTestHelper.createSnapshot(dfs, foo, "s2"); - dfs.delete(f1, true); - - byte[][] f1Components = FSDirectory.getPathComponentsWithSnapshot(f1Node); - byte[][] f2Components = FSDirectory.getPathComponentsWithSnapshot(f2Node); - // expected: /foo/.snapshot/s2/f1 - String f1Snapshot = SnapshotTestHelper.getSnapshotPath(foo, "s2", - f1.getName()).toString(); - // expected: /foo/.snapshot/s1/bar/f2 - String f2Snapshot = SnapshotTestHelper.getSnapshotPath(foo, "s1", "bar/f2") - .toString(); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(f1Snapshot), - f1Components); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(f2Snapshot), - f2Components); - - // delete snapshot s2 - dfs.deleteSnapshot(foo, "s2"); - // expected: /foo/.snapshot/s1/f1 - f1Snapshot = SnapshotTestHelper.getSnapshotPath(foo, "s1", f1.getName()) - .toString(); - f1Components = FSDirectory.getPathComponentsWithSnapshot(f1Node); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(f1Snapshot), - f1Components); - } - - /** - * INode after renaming - */ - @Test - public void testRenamedINode() throws Exception { - final Path foo = new Path("/foo"); - final Path bar = new Path(foo, "bar"); - final Path f1 = new Path(bar, "f1"); - final Path f2 = new Path(bar, "f2"); - DFSTestUtil.createFile(dfs, f1, BLOCKSIZE, REPLICATION, 0L); - DFSTestUtil.createFile(dfs, f2, BLOCKSIZE, REPLICATION, 0L); - - // create snapshot s1 - SnapshotTestHelper.createSnapshot(dfs, foo, "s1"); - INode f2Node = fsdir.getINode4Write(f2.toString()); - // delete /foo/bar/f2 - dfs.delete(f2, true); - // rename bar to bar2 - final Path bar2 = new Path(foo, "bar2"); - dfs.rename(bar, bar2); - // create snapshot s2 - SnapshotTestHelper.createSnapshot(dfs, foo, "s2"); - - // /foo/.snapshot/s1/bar - Path barPath = SnapshotTestHelper.getSnapshotPath(foo, "s1", bar.getName()); - INode barNode = fsdir.getINode(barPath.toString()); - Assert.assertTrue(barNode instanceof INodeReference.WithName); - INode bar2Node = fsdir.getINode(bar2.toString()); - Assert.assertTrue(bar2Node instanceof INodeReference.DstReference); - byte[][] barComponents = FSDirectory.getPathComponentsWithSnapshot(barNode); - byte[][] bar2Components = FSDirectory - .getPathComponentsWithSnapshot(bar2Node); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(barPath.toString()), barComponents); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(bar2.toString()), - bar2Components); - - byte[][] f2Components = FSDirectory.getPathComponentsWithSnapshot(f2Node); - // expected: /foo/.snapshot/s1/bar/f2 - Path deletedf2 = SnapshotTestHelper.getSnapshotPath(foo, "s1", "bar/f2"); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(deletedf2.toString()), f2Components); - - final Path newf1 = new Path(bar2, f1.getName()); - INode f1Node = fsdir.getINode(newf1.toString()); - Assert.assertTrue(dfs.delete(newf1, true)); - Path deletedf1 = SnapshotTestHelper.getSnapshotPath(foo, "s2", "bar2/f1"); - byte[][] f1Components = FSDirectory.getPathComponentsWithSnapshot(f1Node); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(deletedf1.toString()), f1Components); - } - - /** - * Similar with testRenamedINode but the rename is across two snapshottable - * directory. - */ - @Test - public void testRenamedINode2() throws Exception { - final Path foo1 = new Path("/foo1"); - final Path foo2 = new Path("/foo2"); - final Path bar = new Path(foo1, "bar"); - final Path f1 = new Path(bar, "f1"); - final Path f2 = new Path(bar, "f2"); - dfs.mkdirs(foo2); - DFSTestUtil.createFile(dfs, f1, BLOCKSIZE, REPLICATION, 0L); - DFSTestUtil.createFile(dfs, f2, BLOCKSIZE, REPLICATION, 0L); - - // create snapshots on foo1 and foo2 - SnapshotTestHelper.createSnapshot(dfs, foo1, "s1"); - SnapshotTestHelper.createSnapshot(dfs, foo2, "s2"); - INode f2Node = fsdir.getINode4Write(f2.toString()); - // delete /foo1/bar/f2 - dfs.delete(f2, true); - // rename bar to bar2 - final Path bar2 = new Path(foo2, "bar2"); - dfs.rename(bar, bar2); - // create snapshot s3 and s4 on foo1 and foo2 - SnapshotTestHelper.createSnapshot(dfs, foo1, "s3"); - SnapshotTestHelper.createSnapshot(dfs, foo2, "s4"); - - // /foo1/.snapshot/s1/bar - Path barPath = SnapshotTestHelper - .getSnapshotPath(foo1, "s1", bar.getName()); - INode barNode = fsdir.getINode(barPath.toString()); - Assert.assertTrue(barNode instanceof INodeReference.WithName); - INode bar2Node = fsdir.getINode(bar2.toString()); - Assert.assertTrue(bar2Node instanceof INodeReference.DstReference); - byte[][] barComponents = FSDirectory.getPathComponentsWithSnapshot(barNode); - byte[][] bar2Components = FSDirectory - .getPathComponentsWithSnapshot(bar2Node); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(barPath.toString()), barComponents); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(bar2.toString()), - bar2Components); - - byte[][] f2Components = FSDirectory.getPathComponentsWithSnapshot(f2Node); - // expected: /foo1/.snapshot/s1/bar/f2 - Path deletedf2 = SnapshotTestHelper.getSnapshotPath(foo1, "s1", "bar/f2"); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(deletedf2.toString()), f2Components); - - final Path newf1 = new Path(bar2, f1.getName()); - INode f1Node = fsdir.getINode(newf1.toString()); - Assert.assertTrue(dfs.delete(newf1, true)); - // /foo2/.snapshot/s4/bar2/f1 - Path deletedf1 = SnapshotTestHelper.getSnapshotPath(foo2, "s4", "bar2/f1"); - byte[][] f1Components = FSDirectory.getPathComponentsWithSnapshot(f1Node); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(deletedf1.toString()), f1Components); - } - - /** - * Rename a directory to its prior descendant - */ - @Test - public void testNestedRename() throws Exception { - final Path sdir1 = new Path("/dir1"); - final Path sdir2 = new Path("/dir2"); - final Path foo = new Path(sdir1, "foo"); - final Path bar = new Path(foo, "bar"); - dfs.mkdirs(bar); - dfs.mkdirs(sdir2); - - SnapshotTestHelper.createSnapshot(dfs, sdir1, "s1"); - - // /dir1/foo/bar -> /dir2/bar - final Path bar2 = new Path(sdir2, "bar"); - dfs.rename(bar, bar2); - - // /dir1/foo -> /dir2/bar/foo - final Path foo2 = new Path(bar2, "foo"); - dfs.rename(foo, foo2); - - // /dir2/bar - INode bar2Node = fsdir.getINode(bar2.toString()); - Assert.assertTrue(bar2Node instanceof INodeReference.DstReference); - byte[][] bar2Components = FSDirectory - .getPathComponentsWithSnapshot(bar2Node); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(bar2.toString()), - bar2Components); - - // /dir1/.snapshot/s1/foo/bar - String oldbar = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", "foo/bar") - .toString(); - INode oldbarNode = fsdir.getINode(oldbar); - Assert.assertTrue(oldbarNode instanceof INodeReference.WithName); - byte[][] oldbarComponents = FSDirectory - .getPathComponentsWithSnapshot(oldbarNode); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(oldbar), - oldbarComponents); - - // /dir2/bar/foo - INode foo2Node = fsdir.getINode(foo2.toString()); - Assert.assertTrue(foo2Node instanceof INodeReference.DstReference); - byte[][] foo2Components = FSDirectory - .getPathComponentsWithSnapshot(foo2Node); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(foo2.toString()), - foo2Components); - - // /dir1/.snapshot/s1/foo - String oldfoo = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", - foo.getName()).toString(); - INode oldfooNode = fsdir.getINode(oldfoo); - Assert.assertTrue(oldfooNode instanceof INodeReference.WithName); - byte[][] oldfooComponents = FSDirectory - .getPathComponentsWithSnapshot(oldfooNode); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(oldfoo), - oldfooComponents); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java index c88aaf2410..2d1fa77ace 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -169,10 +169,11 @@ public class TestRenameWithSnapshots { } private static boolean existsInDiffReport(List entries, - DiffType type, String relativePath) { + DiffType type, String sourcePath, String targetPath) { for (DiffReportEntry entry : entries) { - if ((entry.getType() == type) - && ((new String(entry.getRelativePath())).compareTo(relativePath) == 0)) { + if (entry.equals(new DiffReportEntry(type, DFSUtil + .string2Bytes(sourcePath), targetPath == null ? null : DFSUtil + .string2Bytes(targetPath)))) { return true; } } @@ -195,8 +196,9 @@ public class TestRenameWithSnapshots { SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); List entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName())); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName(), + null)); } /** @@ -215,10 +217,10 @@ public class TestRenameWithSnapshots { SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); System.out.println("DiffList is " + diffReport.toString()); List entries = diffReport.getDiffList(); - assertTrue(entries.size() == 3); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, file1.getName())); + assertTrue(entries.size() == 2); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), + file2.getName())); } @Test (timeout=60000) @@ -238,26 +240,26 @@ public class TestRenameWithSnapshots { diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, snap2); LOG.info("DiffList is " + diffReport.toString()); List entries = diffReport.getDiffList(); - assertTrue(entries.size() == 3); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, file1.getName())); + assertTrue(entries.size() == 2); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), + file2.getName())); diffReport = hdfs.getSnapshotDiffReport(sub1, snap2, ""); LOG.info("DiffList is " + diffReport.toString()); entries = diffReport.getDiffList(); - assertTrue(entries.size() == 3); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file3.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, file2.getName())); + assertTrue(entries.size() == 2); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, file2.getName(), + file3.getName())); diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); LOG.info("DiffList is " + diffReport.toString()); entries = diffReport.getDiffList(); - assertTrue(entries.size() == 3); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file3.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, file1.getName())); + assertTrue(entries.size() == 2); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), + file3.getName())); } @Test (timeout=60000) @@ -280,11 +282,10 @@ public class TestRenameWithSnapshots { ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List entries = diffReport.getDiffList(); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName())); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, sub2.getName() - + "/" + sub2file2.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, sub2.getName() - + "/" + sub2file1.getName())); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName(), + null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName() + + "/" + sub2file1.getName(), sub2.getName() + "/" + sub2file2.getName())); } @Test (timeout=60000) @@ -307,10 +308,10 @@ public class TestRenameWithSnapshots { ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List entries = diffReport.getDiffList(); - assertEquals(3, entries.size()); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, sub3.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, sub2.getName())); + assertEquals(2, entries.size()); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName(), + sub3.getName())); } /** @@ -2406,12 +2407,12 @@ public class TestRenameWithSnapshots { LOG.info("DiffList is \n\"" + report.toString() + "\""); List entries = report.getDiffList(); assertEquals(7, entries.size()); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, foo.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, bar.getName())); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, newDir.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file1")); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file2")); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file3")); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, foo.getName(), null)); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, bar.getName(), null)); + assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file1", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, "bar", "newDir")); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file2", "newDir/file2")); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file3", "newDir/file1")); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java index 8ba3ef4ecf..0cc318b5ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.HashMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DFSUtil; @@ -143,7 +144,7 @@ public class TestSnapshotDiffReport { hdfs.createSnapshot(snapshotDir, genSnapshotName(snapshotDir)); } // modify file10 - hdfs.setReplication(file10, (short) (REPLICATION - 1)); + hdfs.setReplication(file10, (short) (REPLICATION + 1)); } /** check the correctness of the diff reports */ @@ -166,11 +167,11 @@ public class TestSnapshotDiffReport { } else if (entry.getType() == DiffType.DELETE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.CREATE, entry.getRelativePath()))); + new DiffReportEntry(DiffType.CREATE, entry.getSourcePath()))); } else if (entry.getType() == DiffType.CREATE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.DELETE, entry.getRelativePath()))); + new DiffReportEntry(DiffType.DELETE, entry.getSourcePath()))); } } } @@ -329,5 +330,166 @@ public class TestSnapshotDiffReport { new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("subsub1"))); } - + + /** + * Rename a directory to its prior descendant, and verify the diff report. + */ + @Test + public void testDiffReportWithRename() throws Exception { + final Path root = new Path("/"); + final Path sdir1 = new Path(root, "dir1"); + final Path sdir2 = new Path(root, "dir2"); + final Path foo = new Path(sdir1, "foo"); + final Path bar = new Path(foo, "bar"); + hdfs.mkdirs(bar); + hdfs.mkdirs(sdir2); + + // create snapshot on root + SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); + + // /dir1/foo/bar -> /dir2/bar + final Path bar2 = new Path(sdir2, "bar"); + hdfs.rename(bar, bar2); + + // /dir1/foo -> /dir2/bar/foo + final Path foo2 = new Path(bar2, "foo"); + hdfs.rename(foo, foo2); + + SnapshotTestHelper.createSnapshot(hdfs, root, "s2"); + // let's delete /dir2 to make things more complicated + hdfs.delete(sdir2, true); + + verifyDiffReport(root, "s1", "s2", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")), + new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/foo"), + DFSUtil.string2Bytes("dir2/bar/foo")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), + new DiffReportEntry(DiffType.MODIFY, + DFSUtil.string2Bytes("dir1/foo/bar")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")), + new DiffReportEntry(DiffType.RENAME, DFSUtil + .string2Bytes("dir1/foo/bar"), DFSUtil.string2Bytes("dir2/bar"))); + } + + /** + * Rename a file/dir outside of the snapshottable dir should be reported as + * deleted. Rename a file/dir from outside should be reported as created. + */ + @Test + public void testDiffReportWithRenameOutside() throws Exception { + final Path root = new Path("/"); + final Path dir1 = new Path(root, "dir1"); + final Path dir2 = new Path(root, "dir2"); + final Path foo = new Path(dir1, "foo"); + final Path fileInFoo = new Path(foo, "file"); + final Path bar = new Path(dir2, "bar"); + final Path fileInBar = new Path(bar, "file"); + DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed); + DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPLICATION, seed); + + // create snapshot on /dir1 + SnapshotTestHelper.createSnapshot(hdfs, dir1, "s0"); + + // move bar into dir1 + final Path newBar = new Path(dir1, "newBar"); + hdfs.rename(bar, newBar); + // move foo out of dir1 into dir2 + final Path newFoo = new Path(dir2, "new"); + hdfs.rename(foo, newFoo); + + SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1"); + verifyDiffReport(dir1, "s0", "s1", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes(newBar + .getName())), + new DiffReportEntry(DiffType.DELETE, + DFSUtil.string2Bytes(foo.getName()))); + } + + /** + * Renaming a file/dir then delete the ancestor dir of the rename target + * should be reported as deleted. + */ + @Test + public void testDiffReportWithRenameAndDelete() throws Exception { + final Path root = new Path("/"); + final Path dir1 = new Path(root, "dir1"); + final Path dir2 = new Path(root, "dir2"); + final Path foo = new Path(dir1, "foo"); + final Path fileInFoo = new Path(foo, "file"); + final Path bar = new Path(dir2, "bar"); + final Path fileInBar = new Path(bar, "file"); + DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed); + DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPLICATION, seed); + + SnapshotTestHelper.createSnapshot(hdfs, root, "s0"); + hdfs.rename(fileInFoo, fileInBar, Rename.OVERWRITE); + SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); + verifyDiffReport(root, "s0", "s1", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2/bar")), + new DiffReportEntry(DiffType.DELETE, DFSUtil + .string2Bytes("dir2/bar/file")), + new DiffReportEntry(DiffType.RENAME, + DFSUtil.string2Bytes("dir1/foo/file"), + DFSUtil.string2Bytes("dir2/bar/file"))); + + // delete bar + hdfs.delete(bar, true); + SnapshotTestHelper.createSnapshot(hdfs, root, "s2"); + verifyDiffReport(root, "s0", "s2", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), + new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("dir2/bar")), + new DiffReportEntry(DiffType.DELETE, + DFSUtil.string2Bytes("dir1/foo/file"))); + } + + @Test + public void testDiffReportWithRenameToNewDir() throws Exception { + final Path root = new Path("/"); + final Path foo = new Path(root, "foo"); + final Path fileInFoo = new Path(foo, "file"); + DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed); + + SnapshotTestHelper.createSnapshot(hdfs, root, "s0"); + final Path bar = new Path(root, "bar"); + hdfs.mkdirs(bar); + final Path fileInBar = new Path(bar, "file"); + hdfs.rename(fileInFoo, fileInBar); + SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); + + verifyDiffReport(root, "s0", "s1", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")), + new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("bar")), + new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo/file"), + DFSUtil.string2Bytes("bar/file"))); + } + + /** + * Rename a file and then append some data to it + */ + @Test + public void testDiffReportWithRenameAndAppend() throws Exception { + final Path root = new Path("/"); + final Path foo = new Path(root, "foo"); + DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, REPLICATION, seed); + + SnapshotTestHelper.createSnapshot(hdfs, root, "s0"); + final Path bar = new Path(root, "bar"); + hdfs.rename(foo, bar); + DFSTestUtil.appendFile(hdfs, bar, 10); // append 10 bytes + SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); + + // we always put modification on the file before rename + verifyDiffReport(root, "s0", "s1", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")), + new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo"), + DFSUtil.string2Bytes("bar"))); + } } \ No newline at end of file