From 7b637a3341f2024470929b653a58e9a88dd036c0 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 13 Mar 2020 19:01:23 -0700 Subject: [PATCH] HDFS-14820. The default 8KB buffer of BlockReaderRemote#newBlockReader#BufferedOutputStream is too big. Contributed by Lisheng Sun. --- .../org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 +- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 228d7ac728..407462c6e7 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -150,7 +150,7 @@ public interface HdfsClientConfigKeys { TimeUnit.DAYS.toMillis(10); // 10 days String DFS_CLIENT_BLOCK_READER_REMOTE_BUFFER_SIZE_KEY = "dfs.client.block.reader.remote.buffer.size"; - int DFS_CLIENT_BLOCK_READER_REMOTE_BUFFER_SIZE_DEFAULT = 8192; + int DFS_CLIENT_BLOCK_READER_REMOTE_BUFFER_SIZE_DEFAULT = 512; String DFS_CLIENT_DEAD_NODE_DETECTION_ENABLED_KEY = "dfs.client.deadnode.detection.enabled"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index ede3dc09c0..9fcc04747d 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4305,13 +4305,11 @@ dfs.client.block.reader.remote.buffer.size - 8192 + 512 - The output stream buffer size of a DFSClient remote read. The buffer default value is 8KB. The buffer includes + The output stream buffer size of a DFSClient remote read. The buffer default value is 512B. The buffer includes only some request parameters that are: block, blockToken, clientName, startOffset, len, verifyChecksum, cachingStrategy. - It is recommended to adjust the value according to the workload, which can reduce unnecessary memory - usage and the frequency of the garbage collection. A value of 512 might be reasonable.