HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to dfs.datanode.stripedread.timeout.millis. Contributed by Andrew Wang.
Change-Id: Ice86c5d46d29e94087c0f186b941d5394e7ac0e6
This commit is contained in:
parent
607bda2a21
commit
82a88b92b4
@ -432,3 +432,6 @@
|
||||
|
||||
HDFS-9097. Erasure coding: update EC command "-s" flag to "-p" when
|
||||
specifying policy. (zhz)
|
||||
|
||||
HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to
|
||||
dfs.datanode.stripedread.timeout.millis. (wang via zhz)
|
||||
|
@ -399,8 +399,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
||||
public static final int DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT = 20;
|
||||
public static final String DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.stripedread.buffer.size";
|
||||
public static final int DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT = 64 * 1024;
|
||||
public static final String DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY = "dfs.datanode.stripedread.threshold.millis";
|
||||
public static final int DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT = 5000; //5s
|
||||
public static final String DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY = "dfs.datanode.stripedread.timeout.millis";
|
||||
public static final int DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT = 5000; //5s
|
||||
public static final String DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_KEY = "dfs.datanode.striped.blockrecovery.threads.size";
|
||||
public static final int DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_DEFAULT = 8;
|
||||
public static final String DFS_DATANODE_DNS_INTERFACE_KEY = "dfs.datanode.dns.interface";
|
||||
|
@ -95,16 +95,16 @@ public final class ErasureCodingWorker {
|
||||
|
||||
private ThreadPoolExecutor STRIPED_BLK_RECOVERY_THREAD_POOL;
|
||||
private ThreadPoolExecutor STRIPED_READ_THREAD_POOL;
|
||||
private final int STRIPED_READ_THRESHOLD_MILLIS;
|
||||
private final int STRIPED_READ_TIMEOUT_MILLIS;
|
||||
private final int STRIPED_READ_BUFFER_SIZE;
|
||||
|
||||
public ErasureCodingWorker(Configuration conf, DataNode datanode) {
|
||||
this.datanode = datanode;
|
||||
this.conf = conf;
|
||||
|
||||
STRIPED_READ_THRESHOLD_MILLIS = conf.getInt(
|
||||
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY,
|
||||
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT);
|
||||
STRIPED_READ_TIMEOUT_MILLIS = conf.getInt(
|
||||
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY,
|
||||
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT);
|
||||
initializeStripedReadThreadPool(conf.getInt(
|
||||
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_KEY,
|
||||
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT));
|
||||
@ -556,7 +556,7 @@ private int[] readMinimumStripedData4Recovery(final int[] success)
|
||||
try {
|
||||
StripingChunkReadResult result =
|
||||
StripedBlockUtil.getNextCompletedStripedRead(
|
||||
readService, futures, STRIPED_READ_THRESHOLD_MILLIS);
|
||||
readService, futures, STRIPED_READ_TIMEOUT_MILLIS);
|
||||
int resultIndex = -1;
|
||||
if (result.state == StripingChunkReadResult.SUCCESSFUL) {
|
||||
resultIndex = result.index;
|
||||
|
@ -203,12 +203,12 @@ public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
|
||||
*/
|
||||
public static StripingChunkReadResult getNextCompletedStripedRead(
|
||||
CompletionService<Void> readService, Map<Future<Void>, Integer> futures,
|
||||
final long threshold) throws InterruptedException {
|
||||
final long timeoutMillis) throws InterruptedException {
|
||||
Preconditions.checkArgument(!futures.isEmpty());
|
||||
Future<Void> future = null;
|
||||
try {
|
||||
if (threshold > 0) {
|
||||
future = readService.poll(threshold, TimeUnit.MILLISECONDS);
|
||||
if (timeoutMillis > 0) {
|
||||
future = readService.poll(timeoutMillis, TimeUnit.MILLISECONDS);
|
||||
} else {
|
||||
future = readService.take();
|
||||
}
|
||||
|
@ -2403,23 +2403,23 @@
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.stripedread.threshold.millis</name>
|
||||
<name>dfs.datanode.stripedread.timeout.millis</name>
|
||||
<value>5000</value>
|
||||
<description>datanode striped read threshold in millisecond.
|
||||
<description>Datanode striped read timeout in milliseconds.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.stripedread.threads</name>
|
||||
<value>20</value>
|
||||
<description>datanode striped read thread pool size.
|
||||
<description>Number of threads used by the Datanode for background recovery work.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.stripedread.buffer.size</name>
|
||||
<value>262144</value>
|
||||
<description>datanode striped read buffer size.
|
||||
<description>Datanode striped read buffer size.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
|
@ -101,7 +101,7 @@ Deployment
|
||||
|
||||
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:
|
||||
|
||||
1. `dfs.datanode.stripedread.threshold.millis` - Timeout for striped reads. Default value is 5000 ms.
|
||||
1. `dfs.datanode.stripedread.timeout.millis` - Timeout for striped reads. Default value is 5000 ms.
|
||||
1. `dfs.datanode.stripedread.threads` - Number of concurrent reader threads. Default value is 20 threads.
|
||||
1. `dfs.datanode.stripedread.buffer.size` - Buffer size for reader service. Default value is 256KB.
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user