diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e2b931ec1b..b2b2176f91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -613,10 +613,19 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY = "dfs.mover.max-no-move-interval";
public static final int DFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; // One minute
+ // SPS related configurations
public static final String DFS_STORAGE_POLICY_SATISFIER_ACTIVATE_KEY =
"dfs.storage.policy.satisfier.activate";
public static final boolean DFS_STORAGE_POLICY_SATISFIER_ACTIVATE_DEFAULT =
true;
+ public static final String DFS_STORAGE_POLICY_SATISFIER_RECHECK_TIMEOUT_MILLIS_KEY =
+ "dfs.storage.policy.satisfier.recheck.timeout.millis";
+ public static final int DFS_STORAGE_POLICY_SATISFIER_RECHECK_TIMEOUT_MILLIS_DEFAULT =
+ 5 * 60 * 1000;
+ public static final String DFS_STORAGE_POLICY_SATISFIER_SELF_RETRY_TIMEOUT_MILLIS_KEY =
+ "dfs.storage.policy.satisfier.self.retry.timeout.millis";
+ public static final int DFS_STORAGE_POLICY_SATISFIER_SELF_RETRY_TIMEOUT_MILLIS_DEFAULT =
+ 30 * 60 * 1000;
public static final String DFS_DATANODE_ADDRESS_KEY = "dfs.datanode.address";
public static final int DFS_DATANODE_DEFAULT_PORT = 9866;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index e9bf9b149b..872261ee7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -479,8 +479,8 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled,
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ACTIVATE_KEY,
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ACTIVATE_DEFAULT);
if (storagePolicyEnabled && spsEnabled) {
- sps = new StoragePolicySatisfier(namesystem,
- storageMovementNeeded, this);
+ sps = new StoragePolicySatisfier(namesystem, storageMovementNeeded, this,
+ conf);
} else {
sps = null;
LOG.warn(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
index ce97075692..042aca38b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
@@ -64,14 +64,14 @@ public class BlockStorageMovementAttemptedItems {
// It might take anywhere between 5 to 10 minutes before
// a request is timed out.
//
- private long checkTimeout = 5 * 60 * 1000; // minimum value
+ private long minCheckTimeout = 5 * 60 * 1000; // minimum value
private BlockStorageMovementNeeded blockStorageMovementNeeded;
- public BlockStorageMovementAttemptedItems(long timeoutPeriod,
+ public BlockStorageMovementAttemptedItems(long recheckTimeout,
long selfRetryTimeout,
BlockStorageMovementNeeded unsatisfiedStorageMovementFiles) {
- if (timeoutPeriod > 0) {
- this.checkTimeout = Math.min(checkTimeout, timeoutPeriod);
+ if (recheckTimeout > 0) {
+ this.minCheckTimeout = Math.min(minCheckTimeout, recheckTimeout);
}
this.selfRetryTimeout = selfRetryTimeout;
@@ -196,7 +196,7 @@ public void run() {
try {
blockStorageMovementResultCheck();
blocksStorageMovementUnReportedItemsCheck();
- Thread.sleep(checkTimeout);
+ Thread.sleep(minCheckTimeout);
} catch (InterruptedException ie) {
LOG.info("BlocksStorageMovementAttemptResultMonitor thread "
+ "is interrupted.", ie);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
index a854bd7233..ee59617ea8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
@@ -27,7 +27,9 @@
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@@ -79,15 +81,18 @@ public class StoragePolicySatisfier implements Runnable {
public StoragePolicySatisfier(final Namesystem namesystem,
final BlockStorageMovementNeeded storageMovementNeeded,
- final BlockManager blkManager) {
+ final BlockManager blkManager, Configuration conf) {
this.namesystem = namesystem;
this.storageMovementNeeded = storageMovementNeeded;
this.blockManager = blkManager;
- // TODO: below selfRetryTimeout and checkTimeout can be configurable later
- // Now, the default values of selfRetryTimeout and checkTimeout are 30mins
- // and 5mins respectively
this.storageMovementsMonitor = new BlockStorageMovementAttemptedItems(
- 5 * 60 * 1000, 30 * 60 * 1000, storageMovementNeeded);
+ conf.getLong(
+ DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_RECHECK_TIMEOUT_MILLIS_KEY,
+ DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_RECHECK_TIMEOUT_MILLIS_DEFAULT),
+ conf.getLong(
+ DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_SELF_RETRY_TIMEOUT_MILLIS_KEY,
+ DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_SELF_RETRY_TIMEOUT_MILLIS_DEFAULT),
+ storageMovementNeeded);
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 54dd8fb32c..fab6ece051 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4509,6 +4509,29 @@
+
+ dfs.storage.policy.satisfier.recheck.timeout.millis
+ 300000
+
+ Blocks storage movements monitor re-check interval in milliseconds.
+ This check will verify whether any blocks storage movement results arrived from DN
+ and also verify if any of file blocks movements not at all reported to DN
+ since dfs.storage.policy.satisfier.self.retry.timeout.
+ The default value is 5 * 60 * 1000 (5 mins)
+
+
+
+
+ dfs.storage.policy.satisfier.self.retry.timeout.millis
+ 1800000
+
+ If any of file related block movements not at all reported by coordinator datanode,
+ then after this timeout(in milliseconds), the item will be added back to movement needed list
+ at namenode which will be retried for block movements.
+ The default value is 30 * 60 * 1000 (30 mins)
+
+
+
dfs.pipeline.ecn
false