big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-13227. Add a method to calculate cumulative diff over multiple snapshots in DirectoryDiffList. Contributed by Shashikant Banerjee

Browse Source
This commit is contained in:
Tsz-Wo Nicholas Sze 2018-03-06 13:35:03 -08:00
parent 871d0d39fa
commit 346caa2095
2 changed files with 38 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
View File

@ -235,6 +235,12 @@ abstract class AbstractINodeDiffList<N extends INode,
return diff == null ? Snapshot.CURRENT_STATE_ID : diff.getSnapshotId(); return diff == null ? Snapshot.CURRENT_STATE_ID : diff.getSnapshotId();
} }
public final int getDiffIndexById(final int snapshotId) {
int diffIndex = diffs.binarySearch(snapshotId);
diffIndex = diffIndex < 0 ? (-diffIndex - 1) : diffIndex;
return diffIndex;
}
final int[] changedBetweenSnapshots(Snapshot from, Snapshot to) { final int[] changedBetweenSnapshots(Snapshot from, Snapshot to) {
if (diffs == null) { if (diffs == null) {
return null; return null;
@ -247,10 +253,10 @@ abstract class AbstractINodeDiffList<N extends INode,
} }
final int size = diffs.size(); final int size = diffs.size();
int earlierDiffIndex = diffs.binarySearch(earlier.getId()); int earlierDiffIndex = getDiffIndexById(earlier.getId());
int laterDiffIndex = later == null ? size int laterDiffIndex = later == null ? size
: diffs.binarySearch(later.getId()); : getDiffIndexById(later.getId());
if (-earlierDiffIndex - 1 == size) { if (earlierDiffIndex == size) {
// if the earlierSnapshot is after the latest SnapshotDiff stored in // if the earlierSnapshot is after the latest SnapshotDiff stored in
// diffs, no modification happened after the earlierSnapshot // diffs, no modification happened after the earlierSnapshot
return null; return null;
@ -260,10 +266,6 @@ abstract class AbstractINodeDiffList<N extends INode,
// before it, no modification happened before the laterSnapshot // before it, no modification happened before the laterSnapshot
return null; return null;
} }
earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1)
: earlierDiffIndex;
laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1)
: laterDiffIndex;
return new int[]{earlierDiffIndex, laterDiffIndex}; return new int[]{earlierDiffIndex, laterDiffIndex};
} }

34
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
View File

@ -228,12 +228,18 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
private List<INode> initChildren() { private List<INode> initChildren() {
if (children == null) { if (children == null) {
final ChildrenDiff combined = new ChildrenDiff(); final ChildrenDiff combined = new ChildrenDiff();
for (DirectoryDiff d = DirectoryDiff.this; d != null; DirectoryDiffList directoryDiffList =
d = d.getPosterior()) { currentDir.getDirectoryWithSnapshotFeature().diffs;
final int diffIndex =
directoryDiffList.getDiffIndexById(getSnapshotId());
List<DirectoryDiff> diffList = directoryDiffList
.getDiffListBetweenSnapshots(diffIndex,
directoryDiffList.asList().size(), currentDir);
for (DirectoryDiff d : diffList) {
combined.combinePosterior(d.diff, null); combined.combinePosterior(d.diff, null);
} }
children = combined.apply2Current(ReadOnlyList.Util.asList( children = combined.apply2Current(ReadOnlyList.Util
currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID))); .asList(currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID)));
} }
return children; return children;
} }
@ -383,6 +389,19 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
} }
return NO_SNAPSHOT_ID; return NO_SNAPSHOT_ID;
} }
/**
* Returns the list of diffs between two indexes corresponding to two
* snapshots.
* @param fromIndex Index of the diff corresponding to the earlier snapshot
* @param toIndex Index of the diff corresponding to the later snapshot
* @param dir The Directory to which the diffList belongs
* @return list of directory diffs
*/
List<DirectoryDiff> getDiffListBetweenSnapshots(int fromIndex, int toIndex,
INodeDirectory dir) {
return asList().getMinListForRange(fromIndex, toIndex, dir);
}
} }
private static Map<INode, INode> cloneDiffList(List<INode> diffList) { private static Map<INode, INode> cloneDiffList(List<INode> diffList) {
@ -678,9 +697,10 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
boolean dirMetadataChanged = false; boolean dirMetadataChanged = false;
INodeDirectoryAttributes dirCopy = null; INodeDirectoryAttributes dirCopy = null;
DiffList<DirectoryDiff> difflist = diffs.asList(); List<DirectoryDiff> difflist = diffs
for (int i = earlierDiffIndex; i < laterDiffIndex; i++) { .getDiffListBetweenSnapshots(earlierDiffIndex, laterDiffIndex,
DirectoryDiff sdiff = difflist.get(i); currentINode);
for (DirectoryDiff sdiff : difflist) {
diff.combinePosterior(sdiff.diff, null); diff.combinePosterior(sdiff.diff, null);
if (!dirMetadataChanged && sdiff.snapshotINode != null) { if (!dirMetadataChanged && sdiff.snapshotINode != null) {
if (dirCopy == null) { if (dirCopy == null) {