diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/PmemVolumeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/PmemVolumeManager.java
index 2d77f7ad96..969d18b62a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/PmemVolumeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/PmemVolumeManager.java
@@ -218,6 +218,8 @@ public final class PmemVolumeManager {
       try {
         File pmemDir = new File(volumes[n]);
         File realPmemDir = verifyIfValidPmemVolume(pmemDir);
+        // Clean up the cache left before, if any.
+        cleanup(realPmemDir);
         this.pmemVolumes.add(realPmemDir.getPath());
         long maxBytes;
         if (maxBytesPerPmem == -1) {
@@ -242,17 +244,20 @@ public final class PmemVolumeManager {
       throw new IOException(
           "At least one valid persistent memory volume is required!");
     }
-    cleanup();
+  }
+
+  void cleanup(File realPmemDir) {
+    try {
+      FileUtils.cleanDirectory(realPmemDir);
+    } catch (IOException e) {
+      LOG.error("Failed to clean up " + realPmemDir.getPath(), e);
+    }
   }
 
   void cleanup() {
     // Remove all files under the volume.
-    for (String pmemDir: pmemVolumes) {
-      try {
-        FileUtils.cleanDirectory(new File(pmemDir));
-      } catch (IOException e) {
-        LOG.error("Failed to clean up " + pmemDir, e);
-      }
+    for (String pmemVolume : pmemVolumes) {
+      cleanup(new File(pmemVolume));
     }
   }