big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17528. FsImageValidation: set txid when saving a new image (#6828)

Browse Source
This commit is contained in:
Tsz-Wo Nicholas Sze 2024-06-19 11:38:17 +08:00 committed by GitHub
parent 9710a8d52f
commit 1e6411c9ec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
View File

@ -977,12 +977,16 @@ private void loadFSImage(File curFile, MD5Hash expectedMd5,
" but expecting " + expectedMd5); " but expecting " + expectedMd5);
} }
long txId = loader.getLoadedImageTxId(); final long txId = setLastAppliedTxId(loader);
LOG.info("Loaded image for txid " + txId + " from " + curFile); LOG.info("Loaded image for txid " + txId + " from " + curFile);
lastAppliedTxId = txId;
storage.setMostRecentCheckpointInfo(txId, curFile.lastModified()); storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
} }
synchronized long setLastAppliedTxId(FSImageFormat.LoaderDelegator loader) {
lastAppliedTxId = loader.getLoadedImageTxId();
return lastAppliedTxId;
}
/** /**
* Save the contents of the FS image to the file. * Save the contents of the FS image to the file.
*/ */
@ -1215,8 +1219,9 @@ public void removeFromCheckpointing(long txid) {
} }
void save(FSNamesystem src, File dst) throws IOException { void save(FSNamesystem src, File dst) throws IOException {
final SaveNamespaceContext context = new SaveNamespaceContext(src, final long txid = getCorrectLastAppliedOrWrittenTxId();
getCorrectLastAppliedOrWrittenTxId(), new Canceler()); LOG.info("save fsimage with txid={} to {}", txid, dst.getAbsolutePath());
final SaveNamespaceContext context = new SaveNamespaceContext(src, txid, new Canceler());
final Storage.StorageDirectory storageDirectory = new Storage.StorageDirectory(dst); final Storage.StorageDirectory storageDirectory = new Storage.StorageDirectory(dst);
Files.createDirectories(storageDirectory.getCurrentDir().toPath()); Files.createDirectories(storageDirectory.getCurrentDir().toPath());
new FSImageSaver(context, storageDirectory, NameNodeFile.IMAGE).run(); new FSImageSaver(context, storageDirectory, NameNodeFile.IMAGE).run();

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
View File

@ -214,6 +214,7 @@ int run(Configuration conf, AtomicInteger errorCount) throws Exception {
initConf(conf); initConf(conf);
// check INodeReference // check INodeReference
NameNode.initMetrics(conf, HdfsServerConstants.NamenodeRole.NAMENODE); // to avoid NPE
final FSNamesystem namesystem = checkINodeReference(conf, errorCount); final FSNamesystem namesystem = checkINodeReference(conf, errorCount);
// check INodeMap // check INodeMap
@ -276,14 +277,16 @@ public void run() {
namesystem.getFSDirectory().writeLock(); namesystem.getFSDirectory().writeLock();
try { try {
loader.load(fsImageFile, false); loader.load(fsImageFile, false);
fsImage.setLastAppliedTxId(loader);
} finally { } finally {
namesystem.getFSDirectory().writeUnlock(); namesystem.getFSDirectory().writeUnlock();
namesystem.writeUnlock("loadImage"); namesystem.writeUnlock("loadImage");
} }
} }
t.cancel(); t.cancel();
Cli.println("Loaded %s %s successfully in %s", Cli.println("Loaded %s %s with txid %d successfully in %s",
FS_IMAGE, fsImageFile, StringUtils.formatTime(now() - loadStart)); FS_IMAGE, fsImageFile, namesystem.getFSImage().getLastAppliedTxId(),
StringUtils.formatTime(now() - loadStart));
return namesystem; return namesystem;
} }