HDFS-11881. NameNode consumes a lot of memory for snapshot diff report generation. Contributed by Manoj Govindassamy.

This commit is contained in:
Wei-Chiu Chuang 2017-06-29 06:38:41 -07:00
parent ea1da39b19
commit 16c8dbde57
3 changed files with 56 additions and 4 deletions

View File

@ -186,6 +186,7 @@
import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ChunkedArrayList;
import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.LimitInputStream; import org.apache.hadoop.util.LimitInputStream;
@ -1455,7 +1456,7 @@ public static SnapshotDiffReport convert(
String toSnapshot = reportProto.getToSnapshot(); String toSnapshot = reportProto.getToSnapshot();
List<SnapshotDiffReportEntryProto> list = reportProto List<SnapshotDiffReportEntryProto> list = reportProto
.getDiffReportEntriesList(); .getDiffReportEntriesList();
List<DiffReportEntry> entries = new ArrayList<>(); List<DiffReportEntry> entries = new ChunkedArrayList<>();
for (SnapshotDiffReportEntryProto entryProto : list) { for (SnapshotDiffReportEntryProto entryProto : list) {
DiffReportEntry entry = convert(entryProto); DiffReportEntry entry = convert(entryProto);
if (entry != null) if (entry != null)
@ -2392,7 +2393,7 @@ public static SnapshotDiffReportProto convert(SnapshotDiffReport report) {
return null; return null;
} }
List<DiffReportEntry> entries = report.getDiffList(); List<DiffReportEntry> entries = report.getDiffList();
List<SnapshotDiffReportEntryProto> entryProtos = new ArrayList<>(); List<SnapshotDiffReportEntryProto> entryProtos = new ChunkedArrayList<>();
for (DiffReportEntry entry : entries) { for (DiffReportEntry entry : entries) {
SnapshotDiffReportEntryProto entryProto = convert(entry); SnapshotDiffReportEntryProto entryProto = convert(entry);
if (entryProto != null) if (entryProto != null)

View File

@ -37,6 +37,7 @@
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.primitives.SignedBytes; import com.google.common.primitives.SignedBytes;
import org.apache.hadoop.util.ChunkedArrayList;
/** /**
* A class describing the difference between snapshots of a snapshottable * A class describing the difference between snapshots of a snapshottable
@ -186,7 +187,7 @@ boolean isFromEarlier() {
* @return A {@link SnapshotDiffReport} describing the difference * @return A {@link SnapshotDiffReport} describing the difference
*/ */
public SnapshotDiffReport generateReport() { public SnapshotDiffReport generateReport() {
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>(); List<DiffReportEntry> diffReportList = new ChunkedArrayList<>();
for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) { for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) {
INode node = drEntry.getKey(); INode node = drEntry.getKey();
byte[][] path = drEntry.getValue(); byte[][] path = drEntry.getValue();
@ -213,7 +214,7 @@ public SnapshotDiffReport generateReport() {
*/ */
private List<DiffReportEntry> generateReport(ChildrenDiff dirDiff, private List<DiffReportEntry> generateReport(ChildrenDiff dirDiff,
byte[][] parentPath, boolean fromEarlier, Map<Long, RenameEntry> renameMap) { byte[][] parentPath, boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
List<DiffReportEntry> list = new ArrayList<DiffReportEntry>(); List<DiffReportEntry> list = new ChunkedArrayList<>();
List<INode> created = dirDiff.getList(ListType.CREATED); List<INode> created = dirDiff.getList(ListType.CREATED);
List<INode> deleted = dirDiff.getList(ListType.DELETED); List<INode> deleted = dirDiff.getList(ListType.DELETED);
byte[][] fullPath = new byte[parentPath.length + 1][]; byte[][] fullPath = new byte[parentPath.length + 1][];

View File

@ -25,6 +25,7 @@
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff; import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
import org.apache.hadoop.util.ChunkedArrayList;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Before; import org.junit.Before;
@ -195,4 +196,53 @@ public void testSnapshotCommandsWithURI()throws Exception {
"Disallowing snaphot on " + path + " succeeded", config); "Disallowing snaphot on " + path + " succeeded", config);
fs.delete(new Path("/Fully/QPath"), true); fs.delete(new Path("/Fully/QPath"), true);
} }
@Test (timeout=60000)
public void testSnapshotDiff()throws Exception {
Configuration config = new HdfsConfiguration();
Path snapDirPath = new Path(fs.getUri().toString() + "/snap_dir");
String snapDir = snapDirPath.toString();
fs.mkdirs(snapDirPath);
DFSTestUtil.DFSAdminRun("-allowSnapshot " + snapDirPath, 0,
"Allowing snaphot on " + snapDirPath + " succeeded", config);
DFSTestUtil.createFile(fs, new Path(snapDirPath, "file1"),
1024, (short) 1, 100);
DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn1", config);
DFSTestUtil.createFile(fs, new Path(snapDirPath, "file2"),
1024, (short) 1, 100);
DFSTestUtil.createFile(fs, new Path(snapDirPath, "file3"),
1024, (short) 1, 100);
DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn2", config);
// verify the snapshot diff using api and command line
SnapshotDiffReport report_s1_s2 =
fs.getSnapshotDiffReport(snapDirPath, "sn1", "sn2");
DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir +
" sn1 sn2", 0, report_s1_s2.toString());
DFSTestUtil.FsShellRun("-renameSnapshot " + snapDirPath + " sn2 sn3",
config);
SnapshotDiffReport report_s1_s3 =
fs.getSnapshotDiffReport(snapDirPath, "sn1", "sn3");
DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir +
" sn1 sn3", 0, report_s1_s3.toString());
// Creating 100 more files so as to force DiffReport generation
// backend ChunkedArrayList to create multiple chunks.
for (int i = 0; i < 100; i++) {
DFSTestUtil.createFile(fs, new Path(snapDirPath, "file_" + i),
1, (short) 1, 100);
}
DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn4", config);
DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir +
" sn1 sn4", 0, null);
DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn1", config);
DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn3", config);
DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn4", config);
DFSTestUtil.DFSAdminRun("-disallowSnapshot " + snapDir, 0,
"Disallowing snaphot on " + snapDirPath + " succeeded", config);
fs.delete(new Path("/Fully/QPath"), true);
}
} }