HDFS-11726. [SPS]: StoragePolicySatisfier should not select same storage type as source and destination in same datanode. Surendra Singh Lilhore.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java

@@ -501,15 +501,20 @@ private boolean findSourceAndTargetToMove(
     // avoid choosing a target which already has this block.
     for (int i = 0; i < sourceWithStorageList.size(); i++) {
       StorageTypeNodePair existingTypeNodePair = sourceWithStorageList.get(i);
-      StorageTypeNodePair chosenTarget = chooseTargetTypeInSameNode(blockInfo,
+
-          existingTypeNodePair.dn, expected);
+      // Check whether the block replica is already placed in the expected
-      if (chosenTarget != null) {
+      // storage type in this source datanode.
-        sourceNodes.add(existingTypeNodePair.dn);
+      if (!expected.contains(existingTypeNodePair.storageType)) {
-        sourceStorageTypes.add(existingTypeNodePair.storageType);
+        StorageTypeNodePair chosenTarget = chooseTargetTypeInSameNode(
-        targetNodes.add(chosenTarget.dn);
+            blockInfo, existingTypeNodePair.dn, expected);
-        targetStorageTypes.add(chosenTarget.storageType);
+        if (chosenTarget != null) {
-        expected.remove(chosenTarget.storageType);
+          sourceNodes.add(existingTypeNodePair.dn);
-        // TODO: We can increment scheduled block count for this node?
+          sourceStorageTypes.add(existingTypeNodePair.storageType);
+          targetNodes.add(chosenTarget.dn);
+          targetStorageTypes.add(chosenTarget.storageType);
+          expected.remove(chosenTarget.storageType);
+          // TODO: We can increment scheduled block count for this node?
+        }
       }
       // To avoid choosing this excludeNodes as targets later
       excludeNodes.add(existingTypeNodePair.dn);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java

@@ -763,6 +763,50 @@ public void testBlockMoveInSameAndRemoteDatanodesWithWARM() throws Exception {
     }
   }
 
+  /**
+   * If replica with expected storage type already exist in source DN then that
+   * DN should be skipped.
+   */
+  @Test(timeout = 300000)
+  public void testSPSWhenReplicaWithExpectedStorageAlreadyAvailableInSource()
+      throws Exception {
+    StorageType[][] diskTypes = new StorageType[][] {
+        {StorageType.DISK, StorageType.ARCHIVE},
+        {StorageType.DISK, StorageType.ARCHIVE},
+        {StorageType.DISK, StorageType.ARCHIVE}};
+
+    try {
+      hdfsCluster = startCluster(config, diskTypes, diskTypes.length,
+          storagesPerDatanode, capacity);
+      dfs = hdfsCluster.getFileSystem();
+      // 1. Write two replica on disk
+      DFSTestUtil.createFile(dfs, new Path(file), DEFAULT_BLOCK_SIZE,
+          (short) 2, 0);
+      // 2. Change policy to COLD, so third replica will be written to ARCHIVE.
+      dfs.setStoragePolicy(new Path(file), "COLD");
+
+      // 3.Change replication factor to 3.
+      dfs.setReplication(new Path(file), (short) 3);
+
+      DFSTestUtil
+          .waitExpectedStorageType(file, StorageType.DISK, 2, 30000, dfs);
+      DFSTestUtil.waitExpectedStorageType(file, StorageType.ARCHIVE, 1, 30000,
+          dfs);
+
+      // 4. Change policy to HOT, so we can move the all block to DISK.
+      dfs.setStoragePolicy(new Path(file), "HOT");
+
+      // 4. Satisfy the policy.
+      dfs.satisfyStoragePolicy(new Path(file));
+
+      // 5. Block should move successfully .
+      DFSTestUtil
+          .waitExpectedStorageType(file, StorageType.DISK, 3, 30000, dfs);
+    } finally {
+      shutdownCluster();
+    }
+  }
+
   /**
    * Tests that movements should not be assigned when there is no space in
    * target DN.