From 1e6411c9ec610f406f71a8d9f54d32a400c26815 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Wed, 19 Jun 2024 11:38:17 +0800 Subject: [PATCH] HDFS-17528. FsImageValidation: set txid when saving a new image (#6828) --- .../apache/hadoop/hdfs/server/namenode/FSImage.java | 13 +++++++++---- .../hdfs/server/namenode/FsImageValidation.java | 7 +++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index f9b796551d..fa321fe85e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -977,12 +977,16 @@ private void loadFSImage(File curFile, MD5Hash expectedMd5, " but expecting " + expectedMd5); } - long txId = loader.getLoadedImageTxId(); + final long txId = setLastAppliedTxId(loader); LOG.info("Loaded image for txid " + txId + " from " + curFile); - lastAppliedTxId = txId; storage.setMostRecentCheckpointInfo(txId, curFile.lastModified()); } + synchronized long setLastAppliedTxId(FSImageFormat.LoaderDelegator loader) { + lastAppliedTxId = loader.getLoadedImageTxId(); + return lastAppliedTxId; + } + /** * Save the contents of the FS image to the file. */ @@ -1215,8 +1219,9 @@ public void removeFromCheckpointing(long txid) { } void save(FSNamesystem src, File dst) throws IOException { - final SaveNamespaceContext context = new SaveNamespaceContext(src, - getCorrectLastAppliedOrWrittenTxId(), new Canceler()); + final long txid = getCorrectLastAppliedOrWrittenTxId(); + LOG.info("save fsimage with txid={} to {}", txid, dst.getAbsolutePath()); + final SaveNamespaceContext context = new SaveNamespaceContext(src, txid, new Canceler()); final Storage.StorageDirectory storageDirectory = new Storage.StorageDirectory(dst); Files.createDirectories(storageDirectory.getCurrentDir().toPath()); new FSImageSaver(context, storageDirectory, NameNodeFile.IMAGE).run(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java index 275be90505..4dac221e4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java @@ -214,6 +214,7 @@ int run(Configuration conf, AtomicInteger errorCount) throws Exception { initConf(conf); // check INodeReference + NameNode.initMetrics(conf, HdfsServerConstants.NamenodeRole.NAMENODE); // to avoid NPE final FSNamesystem namesystem = checkINodeReference(conf, errorCount); // check INodeMap @@ -276,14 +277,16 @@ public void run() { namesystem.getFSDirectory().writeLock(); try { loader.load(fsImageFile, false); + fsImage.setLastAppliedTxId(loader); } finally { namesystem.getFSDirectory().writeUnlock(); namesystem.writeUnlock("loadImage"); } } t.cancel(); - Cli.println("Loaded %s %s successfully in %s", - FS_IMAGE, fsImageFile, StringUtils.formatTime(now() - loadStart)); + Cli.println("Loaded %s %s with txid %d successfully in %s", + FS_IMAGE, fsImageFile, namesystem.getFSImage().getLastAppliedTxId(), + StringUtils.formatTime(now() - loadStart)); return namesystem; }