big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17528. FsImageValidation: set txid when saving a new image (#6828)

Browse Source
This commit is contained in:
Tsz-Wo Nicholas Sze 2024-06-19 11:38:17 +08:00 committed by GitHub
parent 9710a8d52f
commit 1e6411c9ec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
View File

@ -977,12 +977,16 @@ private void loadFSImage(File curFile, MD5Hash expectedMd5,
" but expecting " + expectedMd5);
}
long txId = loader.getLoadedImageTxId();
final long txId = setLastAppliedTxId(loader);
LOG.info("Loaded image for txid " + txId + " from " + curFile);
lastAppliedTxId = txId;
storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
}
synchronized long setLastAppliedTxId(FSImageFormat.LoaderDelegator loader) {
lastAppliedTxId = loader.getLoadedImageTxId();
return lastAppliedTxId;
}
/**
* Save the contents of the FS image to the file.
*/
@ -1215,8 +1219,9 @@ public void removeFromCheckpointing(long txid) {
}
void save(FSNamesystem src, File dst) throws IOException {
final SaveNamespaceContext context = new SaveNamespaceContext(src,
getCorrectLastAppliedOrWrittenTxId(), new Canceler());
final long txid = getCorrectLastAppliedOrWrittenTxId();
LOG.info("save fsimage with txid={} to {}", txid, dst.getAbsolutePath());
final SaveNamespaceContext context = new SaveNamespaceContext(src, txid, new Canceler());
final Storage.StorageDirectory storageDirectory = new Storage.StorageDirectory(dst);
Files.createDirectories(storageDirectory.getCurrentDir().toPath());
new FSImageSaver(context, storageDirectory, NameNodeFile.IMAGE).run();

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
View File

@ -214,6 +214,7 @@ int run(Configuration conf, AtomicInteger errorCount) throws Exception {
initConf(conf);
// check INodeReference
NameNode.initMetrics(conf, HdfsServerConstants.NamenodeRole.NAMENODE); // to avoid NPE
final FSNamesystem namesystem = checkINodeReference(conf, errorCount);
// check INodeMap
@ -276,14 +277,16 @@ public void run() {
namesystem.getFSDirectory().writeLock();
try {
loader.load(fsImageFile, false);
fsImage.setLastAppliedTxId(loader);
} finally {
namesystem.getFSDirectory().writeUnlock();
namesystem.writeUnlock("loadImage");
}
}
t.cancel();
Cli.println("Loaded %s %s successfully in %s",
FS_IMAGE, fsImageFile, StringUtils.formatTime(now() - loadStart));
Cli.println("Loaded %s %s with txid %d successfully in %s",
FS_IMAGE, fsImageFile, namesystem.getFSImage().getLastAppliedTxId(),
StringUtils.formatTime(now() - loadStart));
return namesystem;
}