big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13813. Exit NameNode if dangling child inode is detected when saving FsImage. Contributed by Siyao Meng.

Browse Source
This commit is contained in:
Wei-Chiu Chuang 2018-08-13 16:12:37 -07:00
parent 74411ce0ce
commit 23854443ef
2 changed files with 32 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
View File

@ -439,6 +439,8 @@ private void loadRootINode(INodeSection.INode p) {
}
public final static class Saver {
private long numImageErrors;
private static long buildPermissionStatus(INodeAttributes n,
final SaverContext.DeduplicationMap<String> stringMap) {
long userId = stringMap.getId(n.getUserName());
@ -563,11 +565,13 @@ public static INodeSection.INodeDirectory.Builder buildINodeDirectory(
this.summary = summary;
this.context = parent.getContext();
this.fsn = context.getSourceNamesystem();
this.numImageErrors = 0;
}
void serializeINodeDirectorySection(OutputStream out) throws IOException {
Iterator<INodeWithAdditionalFields> iter = fsn.getFSDirectory()
.getINodeMap().getMapIterator();
FSDirectory dir = fsn.getFSDirectory();
Iterator<INodeWithAdditionalFields> iter = dir.getINodeMap()
.getMapIterator();
final ArrayList<INodeReference> refList = parent.getSaverContext()
.getRefList();
int i = 0;
@ -583,6 +587,17 @@ void serializeINodeDirectorySection(OutputStream out) throws IOException {
INodeDirectorySection.DirEntry.Builder b = INodeDirectorySection.
DirEntry.newBuilder().setParent(n.getId());
for (INode inode : children) {
// Error if the child inode doesn't exist in inodeMap
if (dir.getInode(inode.getId()) == null) {
FSImage.LOG.error(
"FSImageFormatPBINode#serializeINodeDirectorySection: " +
"Dangling child pointer found. Missing INode in " +
"inodeMap: id=" + inode.getId() +
"; path=" + inode.getFullPathName() +
"; parent=" + (inode.getParent() == null ? "null" :
inode.getParent().getFullPathName()));
++numImageErrors;
}
if (!inode.isReference()) {
b.addChildren(inode.getId());
} else {
@ -711,6 +726,15 @@ private INodeSection.INode.Builder buildINodeCommon(INode n) {
.setId(n.getId())
.setName(ByteString.copyFrom(n.getLocalNameBytes()));
}
/**
* Number of non-fatal errors detected while writing the
* INodeSection and INodeDirectorySection sections.
* @return the number of non-fatal errors detected.
*/
public long getNumImageErrors() {
return numImageErrors;
}
}
private FSImageFormatPBINode() {

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
View File

@ -474,13 +474,15 @@ private static void saveFileSummary(OutputStream out, FileSummary summary)
out.write(lengthBytes);
}
private void saveInodes(FileSummary.Builder summary) throws IOException {
private long saveInodes(FileSummary.Builder summary) throws IOException {
FSImageFormatPBINode.Saver saver = new FSImageFormatPBINode.Saver(this,
summary);
saver.serializeINodeSection(sectionOutputStream);
saver.serializeINodeDirectorySection(sectionOutputStream);
saver.serializeFilesUCSection(sectionOutputStream);
return saver.getNumImageErrors();
}
/**
@ -543,8 +545,9 @@ private long saveInternal(FileOutputStream fout,
step = new Step(StepType.INODES, filePath);
prog.beginStep(Phase.SAVING_CHECKPOINT, step);
saveInodes(b);
long numErrors = saveSnapshots(b);
// Count number of non-fatal errors when saving inodes and snapshots.
long numErrors = saveInodes(b);
numErrors += saveSnapshots(b);
prog.endStep(Phase.SAVING_CHECKPOINT, step);
step = new Step(StepType.DELEGATION_TOKENS, filePath);