big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11881. NameNode consumes a lot of memory for snapshot diff report generation. Contributed by Manoj Govindassamy.

Browse Source
This commit is contained in:
Wei-Chiu Chuang 2017-06-29 06:38:41 -07:00
parent ea1da39b19
commit 16c8dbde57
3 changed files with 56 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
View File

@ -186,6 +186,7 @@
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ChunkedArrayList;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.LimitInputStream;
@ -1455,7 +1456,7 @@ public static SnapshotDiffReport convert(
String toSnapshot = reportProto.getToSnapshot();
List<SnapshotDiffReportEntryProto> list = reportProto
.getDiffReportEntriesList();
List<DiffReportEntry> entries = new ArrayList<>();
List<DiffReportEntry> entries = new ChunkedArrayList<>();
for (SnapshotDiffReportEntryProto entryProto : list) {
DiffReportEntry entry = convert(entryProto);
if (entry != null)
@ -2392,7 +2393,7 @@ public static SnapshotDiffReportProto convert(SnapshotDiffReport report) {
return null;
}
List<DiffReportEntry> entries = report.getDiffList();
List<SnapshotDiffReportEntryProto> entryProtos = new ArrayList<>();
List<SnapshotDiffReportEntryProto> entryProtos = new ChunkedArrayList<>();
for (DiffReportEntry entry : entries) {
SnapshotDiffReportEntryProto entryProto = convert(entry);
if (entryProto != null)

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
View File

@ -37,6 +37,7 @@
import com.google.common.base.Preconditions;
import com.google.common.primitives.SignedBytes;
import org.apache.hadoop.util.ChunkedArrayList;
/**
* A class describing the difference between snapshots of a snapshottable
@ -186,7 +187,7 @@ boolean isFromEarlier() {
* @return A {@link SnapshotDiffReport} describing the difference
*/
public SnapshotDiffReport generateReport() {
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
List<DiffReportEntry> diffReportList = new ChunkedArrayList<>();
for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) {
INode node = drEntry.getKey();
byte[][] path = drEntry.getValue();
@ -213,7 +214,7 @@ public SnapshotDiffReport generateReport() {
*/
private List<DiffReportEntry> generateReport(ChildrenDiff dirDiff,
byte[][] parentPath, boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
List<DiffReportEntry> list = new ChunkedArrayList<>();
List<INode> created = dirDiff.getList(ListType.CREATED);
List<INode> deleted = dirDiff.getList(ListType.DELETED);
byte[][] fullPath = new byte[parentPath.length + 1][];

50
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
View File

@ -25,6 +25,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
import org.apache.hadoop.util.ChunkedArrayList;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@ -195,4 +196,53 @@ public void testSnapshotCommandsWithURI()throws Exception {
"Disallowing snaphot on " + path + " succeeded", config);
fs.delete(new Path("/Fully/QPath"), true);
}
@Test (timeout=60000)
public void testSnapshotDiff()throws Exception {
Configuration config = new HdfsConfiguration();
Path snapDirPath = new Path(fs.getUri().toString() + "/snap_dir");
String snapDir = snapDirPath.toString();
fs.mkdirs(snapDirPath);
DFSTestUtil.DFSAdminRun("-allowSnapshot " + snapDirPath, 0,
"Allowing snaphot on " + snapDirPath + " succeeded", config);
DFSTestUtil.createFile(fs, new Path(snapDirPath, "file1"),
1024, (short) 1, 100);
DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn1", config);
DFSTestUtil.createFile(fs, new Path(snapDirPath, "file2"),
1024, (short) 1, 100);
DFSTestUtil.createFile(fs, new Path(snapDirPath, "file3"),
1024, (short) 1, 100);
DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn2", config);
// verify the snapshot diff using api and command line
SnapshotDiffReport report_s1_s2 =
fs.getSnapshotDiffReport(snapDirPath, "sn1", "sn2");
DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir +
" sn1 sn2", 0, report_s1_s2.toString());
DFSTestUtil.FsShellRun("-renameSnapshot " + snapDirPath + " sn2 sn3",
config);
SnapshotDiffReport report_s1_s3 =
fs.getSnapshotDiffReport(snapDirPath, "sn1", "sn3");
DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir +
" sn1 sn3", 0, report_s1_s3.toString());
// Creating 100 more files so as to force DiffReport generation
// backend ChunkedArrayList to create multiple chunks.
for (int i = 0; i < 100; i++) {
DFSTestUtil.createFile(fs, new Path(snapDirPath, "file_" + i),
1, (short) 1, 100);
}
DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn4", config);
DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir +
" sn1 sn4", 0, null);
DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn1", config);
DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn3", config);
DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn4", config);
DFSTestUtil.DFSAdminRun("-disallowSnapshot " + snapDir, 0,
"Disallowing snaphot on " + snapDirPath + " succeeded", config);
fs.delete(new Path("/Fully/QPath"), true);
}
}