big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17010. Add a subtree test to TestSnapshotDiffReport. (#5656)

Browse Source
This commit is contained in:
Tsz-Wo Nicholas Sze 2023-05-18 15:53:26 +08:00 committed by GitHub
parent 03163f9de2
commit 350dcaf616
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 103 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapRootDescendantDiff.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -19,10 +19,13 @@
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
@ -32,7 +35,19 @@
/**
* Test snapshot diff report for the snapshot root descendant directory.
*/
public class TestSnapRootDescendantDiff extends TestSnapshotDiffReport {
public class TestSnapRootDescendantDiff {
{
SnapshotTestHelper.disableLogs();
}
private final Path dir = new Path("/" + getClass().getSimpleName());
private final Path sub1 = new Path(dir, "sub1");
protected Configuration conf;
protected MiniDFSCluster cluster;
protected DistributedFileSystem hdfs;
private final Map<Path, Integer> snapshotNumberMap = new HashMap<>();
@Before
public void setUp() throws Exception {
conf = new Configuration();
@ -59,6 +74,25 @@ public void tearDown() throws Exception {
}
}
private Path getSnapRootDir() {
return sub1;
}
private String genSnapshotName(Path snapshotDir) {
int sNum = -1;
if (snapshotNumberMap.containsKey(snapshotDir)) {
sNum = snapshotNumberMap.get(snapshotDir);
}
snapshotNumberMap.put(snapshotDir, ++sNum);
return "s" + sNum;
}
void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
throws Exception {
TestSnapshotDiffReport.modifyAndCreateSnapshot(
modifyDir, snapshotDirs, hdfs, this::genSnapshotName);
}
@Test
public void testNonSnapRootDiffReport() throws Exception {
Path subsub1 = new Path(getSnapRootDir(), "subsub1");

88
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -30,6 +30,7 @@
import java.util.Random;
import java.util.List;
import java.util.ArrayList;
import java.util.function.Function;
import org.apache.commons.collections.list.TreeList;
import org.apache.hadoop.conf.Configuration;
@ -50,6 +51,8 @@
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@ -58,7 +61,6 @@
import org.apache.hadoop.util.Time;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@ -70,6 +72,10 @@
public class TestSnapshotDiffReport {
private static final Logger LOG =
LoggerFactory.getLogger(TestSnapshotDiffReport.class);
{
SnapshotTestHelper.disableLogs();
}
private static final long SEED = 0;
private static final short REPLICATION = 3;
private static final short REPLICATION_1 = 2;
@ -80,9 +86,9 @@ public class TestSnapshotDiffReport {
private final Path dir = new Path("/TestSnapshot");
private final Path sub1 = new Path(dir, "sub1");
protected Configuration conf;
protected MiniDFSCluster cluster;
protected DistributedFileSystem hdfs;
private Configuration conf;
private MiniDFSCluster cluster;
private DistributedFileSystem hdfs;
private final HashMap<Path, Integer> snapshotNumberMap = new HashMap<Path, Integer>();
@Before
@ -112,10 +118,6 @@ public void tearDown() throws Exception {
}
}
protected Path getSnapRootDir() {
return sub1;
}
private String genSnapshotName(Path snapshotDir) {
int sNum = -1;
if (snapshotNumberMap.containsKey(snapshotDir)) {
@ -125,11 +127,16 @@ private String genSnapshotName(Path snapshotDir) {
return "s" + sNum;
}
void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
throws Exception {
modifyAndCreateSnapshot(modifyDir, snapshotDirs, hdfs, this::genSnapshotName);
}
/**
* Create/modify/delete files under a given directory, also create snapshots
* of directories.
*/
protected void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
static void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs,
DistributedFileSystem hdfs, Function<Path, String> getSnapshotName)
throws Exception {
Path file10 = new Path(modifyDir, "file10");
Path file11 = new Path(modifyDir, "file11");
@ -147,7 +154,7 @@ protected void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
// create snapshot
for (Path snapshotDir : snapshotDirs) {
hdfs.allowSnapshot(snapshotDir);
hdfs.createSnapshot(snapshotDir, genSnapshotName(snapshotDir));
hdfs.createSnapshot(snapshotDir, getSnapshotName.apply(snapshotDir));
}
// delete file11
@ -165,7 +172,7 @@ protected void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
// create snapshot
for (Path snapshotDir : snapshotDirs) {
hdfs.createSnapshot(snapshotDir, genSnapshotName(snapshotDir));
hdfs.createSnapshot(snapshotDir, getSnapshotName.apply(snapshotDir));
}
// create file11 again
@ -183,7 +190,7 @@ protected void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
// create snapshot
for (Path snapshotDir : snapshotDirs) {
hdfs.createSnapshot(snapshotDir, genSnapshotName(snapshotDir));
hdfs.createSnapshot(snapshotDir, getSnapshotName.apply(snapshotDir));
}
// modify file10
hdfs.setReplication(file10, (short) (REPLICATION + 1));
@ -317,10 +324,6 @@ public void testDiffReport() throws Exception {
@Test(timeout = 60000)
public void testSnapRootDescendantDiffReport() throws Exception {
Assume.assumeTrue(conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT,
DFSConfigKeys.
DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT_DEFAULT));
Path subSub = new Path(sub1, "subsub1");
Path subSubSub = new Path(subSub, "subsubsub1");
Path nonSnapDir = new Path(dir, "non_snap");
@ -579,10 +582,6 @@ private void verifyDescendantDiffReports(final Path snapDir,
@Test
public void testSnapRootDescendantDiffReportWithRename() throws Exception {
Assume.assumeTrue(conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT,
DFSConfigKeys.
DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT_DEFAULT));
Path subSub = new Path(sub1, "subsub1");
Path subSubSub = new Path(subSub, "subsubsub1");
Path nonSnapDir = new Path(dir, "non_snap");
@ -1603,4 +1602,51 @@ public void testSnapshotDiffReportRemoteIterator2() throws Exception {
+ "supported for snapshotDiffReport between two snapshots"));
}
}
@Test
public void testSubtrees() throws Exception {
final Path root = new Path("/");
final Path foo = new Path(root, "foo");
final Path bar = new Path(foo, "bar");
hdfs.mkdirs(bar);
modifyAndCreateSnapshot(bar, new Path[]{root});
final SnapshottableDirectoryStatus[] snapshottables
= hdfs.getSnapshottableDirListing();
Assert.assertEquals(1, snapshottables.length);
Assert.assertEquals(3, snapshottables[0].getSnapshotNumber());
final SnapshotStatus[] statuses = hdfs.getSnapshotListing(root);
Assert.assertEquals(3, statuses.length);
for (int i = 0; i < statuses.length; i++) {
final SnapshotStatus s = statuses[i];
LOG.info("Snapshot #{}: {}", s.getSnapshotID(), s.getFullPath());
Assert.assertEquals(i, s.getSnapshotID());
}
for (int i = 0; i <= 2; i++) {
for (int j = 0; j <= 2; j++) {
assertDiff(root, foo, bar, "s" + i, "s" + j);
}
}
}
private void assertDiff(Path root, Path foo, Path bar,
String from, String to) throws Exception {
final String barDiff = diff(bar, from, to);
final String fooDiff = diff(foo, from, to);
Assert.assertEquals(barDiff, fooDiff.replace("/bar", ""));
final String rootDiff = diff(root, from, to);
Assert.assertEquals(fooDiff, rootDiff.replace("/foo", ""));
Assert.assertEquals(barDiff, rootDiff.replace("/foo/bar", ""));
}
private String diff(Path path, String from, String to) throws Exception {
final SnapshotDiffReport diff = hdfs.getSnapshotDiffReport(path, from, to);
LOG.info("DIFF {} from {} to {}", path, from, to);
LOG.info("{}", diff);
final String report = diff.toString();
return report.substring(report.indexOf(":") + 1);
}
}