big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15500. In-order deletion of snapshots: Diff lists must be update only in the last snapshot. (#2233)

Browse Source
This commit is contained in:
Tsz-Wo Nicholas Sze 2020-08-27 02:24:52 -07:00 committed by GitHub
parent d8aaa8c338
commit 41182a9b6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 27 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -1569,6 +1569,12 @@ public void checkOperation(OperationCategory op) throws StandbyException {
// null in some unit tests
haContext.checkOperation(op);
}
boolean assertsEnabled = false;
assert assertsEnabled = true; // Intentional side effect!!!
if (assertsEnabled && op == OperationCategory.WRITE) {
getSnapshotManager().initThreadLocals();
}
}
/**

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
View File

@ -76,7 +76,11 @@ public final void deleteSnapshotDiff(INode.ReclaimContext reclaimContext,
if (diffs == null) {
return;
}
int snapshotIndex = diffs.binarySearch(snapshot);
final int snapshotIndex = diffs.binarySearch(snapshot);
// DeletionOrdered: only can remove the element at index 0 and no prior
// check snapshotIndex <= 0 since the diff may not exist
assert !SnapshotManager.isDeletionOrdered()
|| (snapshotIndex <= 0 && prior == Snapshot.NO_SNAPSHOT_ID);
D removed;
if (snapshotIndex == 0) {

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java
View File

@ -57,6 +57,8 @@ public int size() {
@Override
public T remove(int i) {
// DeletionOrdered: only can remove the element at index 0
assert !SnapshotManager.isDeletionOrdered() || i == 0;
return list.remove(i);
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
View File

@ -175,6 +175,8 @@ void combinePosteriorAndCollectBlocks(
final INode.ReclaimContext reclaimContext,
final INodeDirectory currentDir,
final DirectoryDiff posterior) {
// DeletionOrdered: must not combine posterior
assert !SnapshotManager.isDeletionOrdered();
diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() {
/** Collect blocks for deleted files. */
@Override

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
View File

@ -95,6 +95,18 @@ public class SnapshotManager implements SnapshotStatsMXBean {
static final long DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT
= 5 * 60_000L; //5 minutes
private static final ThreadLocal<Boolean> DELETION_ORDERED
= new ThreadLocal<>();
static boolean isDeletionOrdered() {
final Boolean b = DELETION_ORDERED.get();
return b != null? b: false;
}
public void initThreadLocals() {
DELETION_ORDERED.set(isSnapshotDeletionOrdered());
}
private final FSDirectory fsdir;
private boolean captureOpenFiles;
/**