big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13315. Add a test for the issue reported in HDFS-11481 which is fixed by HDFS-10997. Contributed by Yongjun Zhang.

Browse Source
This commit is contained in:
Yongjun Zhang 2018-03-20 23:00:39 -07:00
parent a3ed6f40e1
commit 6c63cc7d30
4 changed files with 80 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
View File

@ -106,6 +106,7 @@ public int run(String[] argv) throws Exception {
} catch (IOException e) {
String[] content = e.getLocalizedMessage().split("\n");
System.err.println("snapshotDiff: " + content[0]);
e.printStackTrace(System.err);
return 1;
}
return 0;

37
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
View File

@ -121,6 +121,9 @@
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@ -2354,4 +2357,38 @@ public static HashSet<Path> closeOpenFiles(
}
return closedFiles;
}
/**
* Check the correctness of the snapshotDiff report.
* Make sure all items in the passed entries are in the snapshotDiff
* report.
*/
public static void verifySnapshotDiffReport(DistributedFileSystem fs,
Path dir, String from, String to,
DiffReportEntry... entries) throws IOException {
SnapshotDiffReport report = fs.getSnapshotDiffReport(dir, from, to);
// reverse the order of from and to
SnapshotDiffReport inverseReport = fs
.getSnapshotDiffReport(dir, to, from);
LOG.info(report.toString());
LOG.info(inverseReport.toString() + "\n");
assertEquals(entries.length, report.getDiffList().size());
assertEquals(entries.length, inverseReport.getDiffList().size());
for (DiffReportEntry entry : entries) {
if (entry.getType() == DiffType.MODIFY) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(entry));
} else if (entry.getType() == DiffType.DELETE) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(
new DiffReportEntry(DiffType.CREATE, entry.getSourcePath())));
} else if (entry.getType() == DiffType.CREATE) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(
new DiffReportEntry(DiffType.DELETE, entry.getSourcePath())));
}
}
}
}

41
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
View File

@ -75,6 +75,8 @@
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.server.namenode.EncryptionFaultInjector;
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
@ -148,6 +150,7 @@
import javax.xml.parsers.SAXParserFactory;
public class TestEncryptionZones {
static final Logger LOG = Logger.getLogger(TestEncryptionZones.class);
protected Configuration conf;
private FileSystemTestHelper fsHelper;
@ -1495,6 +1498,44 @@ public void testSnapshotWithFile() throws Exception {
contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));
}
/**
* Check the correctness of the diff reports.
*/
private void verifyDiffReport(Path dir, String from, String to,
DiffReportEntry... entries) throws IOException {
DFSTestUtil.verifySnapshotDiffReport(fs, dir, from, to, entries);
}
/**
* Test correctness of snapshotDiff for encryption zone.
* snapshtoDiff should work when the path parameter is prefixed with
* /.reserved/raw for path that's both snapshottable and encryption zone.
*/
@Test
public void testSnapshotDiffOnEncryptionZones() throws Exception {
final String TEST_KEY2 = "testkey2";
DFSTestUtil.createKey(TEST_KEY2, cluster, conf);
final int len = 8196;
final Path zone = new Path("/zone");
final Path rawZone = new Path("/.reserved/raw/zone");
final Path zoneFile = new Path(zone, "zoneFile");
fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
dfsAdmin.allowSnapshot(zone);
dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
DFSTestUtil.createFile(fs, zoneFile, len, (short) 1, 0xFEED);
fs.createSnapshot(zone, "snap1");
fsWrapper.delete(zoneFile, true);
fs.createSnapshot(zone, "snap2");
verifyDiffReport(zone, "snap1", "snap2",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("zoneFile")));
verifyDiffReport(rawZone, "snap1", "snap2",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("zoneFile")));
}
/**
* Verify symlinks can be created in encryption zones and that
* they function properly when the target is in the same

25
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
View File

@ -194,30 +194,7 @@ protected void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
*/
private void verifyDiffReport(Path dir, String from, String to,
DiffReportEntry... entries) throws IOException {
SnapshotDiffReport report = hdfs.getSnapshotDiffReport(dir, from, to);
// reverse the order of from and to
SnapshotDiffReport inverseReport = hdfs
.getSnapshotDiffReport(dir, to, from);
LOG.info(report.toString());
LOG.info(inverseReport.toString() + "\n");
assertEquals(entries.length, report.getDiffList().size());
assertEquals(entries.length, inverseReport.getDiffList().size());
for (DiffReportEntry entry : entries) {
if (entry.getType() == DiffType.MODIFY) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(entry));
} else if (entry.getType() == DiffType.DELETE) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(
new DiffReportEntry(DiffType.CREATE, entry.getSourcePath())));
} else if (entry.getType() == DiffType.CREATE) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(
new DiffReportEntry(DiffType.DELETE, entry.getSourcePath())));
}
}
DFSTestUtil.verifySnapshotDiffReport(hdfs, dir, from, to, entries);
}
/**