From 88cce32551e6d52fd1c5a5bfd6c41499bf6ab1ab Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Fri, 19 Oct 2018 10:38:04 -0700 Subject: [PATCH] HDFS-9872. HDFS bytes-default configurations should accept multiple size units. Contributed by Yiqun Lin. --- .../org/apache/hadoop/hdfs/DFSClient.java | 2 +- .../hadoop/hdfs/server/datanode/DNConf.java | 2 +- .../AvailableSpaceVolumeChoosingPolicy.java | 2 +- .../impl/ReservedSpaceCalculator.java | 2 +- .../hdfs/server/namenode/FSDirectory.java | 4 +-- .../hdfs/server/namenode/FSNamesystem.java | 3 +- .../hdfs/server/namenode/ImageServlet.java | 8 ++--- .../namenode/NameNodeResourceChecker.java | 2 +- .../hdfs/server/namenode/TransferFsImage.java | 2 +- .../src/main/resources/hdfs-default.xml | 32 ++++++++++++++----- 10 files changed, 38 insertions(+), 21 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index ce1083deb7..2badbb14b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -371,7 +371,7 @@ public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode, (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_READS) == null) ? null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_READS, false); Long readahead = (conf.get(DFS_CLIENT_CACHE_READAHEAD) == null) ? - null : conf.getLong(DFS_CLIENT_CACHE_READAHEAD, 0); + null : conf.getLongBytes(DFS_CLIENT_CACHE_READAHEAD, 0); this.serverDefaultsValidityPeriod = conf.getLong(DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY, DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java index 8e5b59743f..d396600a6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java @@ -248,7 +248,7 @@ public DNConf(final Configurable dn) { DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT); - this.maxLockedMemory = getConf().getLong( + this.maxLockedMemory = getConf().getLongBytes( DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java index 67a66fd0ff..72ed47c67f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java @@ -76,7 +76,7 @@ private void initLocks() { @Override public void setConf(Configuration conf) { - balancedSpaceThreshold = conf.getLong( + balancedSpaceThreshold = conf.getLongBytes( DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_KEY, DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT); balancedPreferencePercent = conf.getFloat( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java index 5523cfdf39..749e16e659 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java @@ -97,7 +97,7 @@ DF getUsage() { long getReservedFromConf(String key, long defaultValue) { return conf.getLong(key + "." + StringUtils.toLowerCase( - storageType.toString()), conf.getLong(key, defaultValue)); + storageType.toString()), conf.getLongBytes(key, defaultValue)); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 2a976d22f4..0140912043 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -291,7 +291,7 @@ public enum DirOp { DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT); LOG.info("XAttrs enabled? " + xattrsEnabled); - this.xattrMaxSize = conf.getInt( + this.xattrMaxSize = (int) conf.getLongBytes( DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY, DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT); Preconditions.checkArgument(xattrMaxSize > 0, @@ -327,7 +327,7 @@ public enum DirOp { DFSConfigKeys.DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_DEFAULT); // filesystem limits - this.maxComponentLength = conf.getInt( + this.maxComponentLength = (int) conf.getLongBytes( DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY, DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT); this.maxDirItems = conf.getInt( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index cc38036759..eda1164f5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -825,7 +825,8 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, DFS_NAMENODE_MAX_OBJECTS_DEFAULT); - this.minBlockSize = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, + this.minBlockSize = conf.getLongBytes( + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT); this.maxBlocksPerFile = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY, DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java index 9028b362e0..7250cca3ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java @@ -220,9 +220,9 @@ public static void setFileNameHeaders(HttpServletResponse response, * @return a data transfer throttler */ public static DataTransferThrottler getThrottler(Configuration conf) { - long transferBandwidth = - conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY, - DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT); + long transferBandwidth = conf.getLongBytes( + DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY, + DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT); DataTransferThrottler throttler = null; if (transferBandwidth > 0) { throttler = new DataTransferThrottler(transferBandwidth); @@ -233,7 +233,7 @@ public static DataTransferThrottler getThrottler(Configuration conf) { private static DataTransferThrottler getThrottlerForBootstrapStandby( Configuration conf) { long transferBandwidth = - conf.getLong( + conf.getLongBytes( DFSConfigKeys.DFS_IMAGE_TRANSFER_BOOTSTRAP_STANDBY_RATE_KEY, DFSConfigKeys.DFS_IMAGE_TRANSFER_BOOTSTRAP_STANDBY_RATE_DEFAULT); DataTransferThrottler throttler = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java index 898f57e4b0..d0245d8035 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java @@ -111,7 +111,7 @@ public NameNodeResourceChecker(Configuration conf) throws IOException { this.conf = conf; volumes = new HashMap(); - duReserved = conf.getLong(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY, + duReserved = conf.getLongBytes(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY, DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_DEFAULT); Collection extraCheckedVolumes = Util.stringCollectionAsURIs(conf diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java index 14ce00098f..c1270c565a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java @@ -275,7 +275,7 @@ private static void uploadImage(URL url, Configuration conf, connection.setDoOutput(true); - int chunkSize = conf.getInt( + int chunkSize = (int) conf.getLongBytes( DFSConfigKeys.DFS_IMAGE_TRANSFER_CHUNKSIZE_KEY, DFSConfigKeys.DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT); if (imageFile.length() > chunkSize) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index cb770f512d..b894abb600 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -348,7 +348,8 @@ corresponding storage types ([ssd]/[disk]/[archive]/[ram_disk]) for cluster with heterogeneous storage. For example, reserved space for RAM_DISK storage can be configured using property 'dfs.datanode.du.reserved.ram_disk'. If specific storage type reservation is not configured - then dfs.datanode.du.reserved will be used. + then dfs.datanode.du.reserved will be used. Support multiple size unit suffix(case insensitive), + as described in dfs.blocksize. Note: In case of using tune2fs to set reserved-blocks-percentage, or other filesystem tools, then you can possibly run into out of disk errors because hadoop will not check those external tool configurations. @@ -390,7 +391,9 @@ dfs.namenode.fs-limits.max-component-length 255 Defines the maximum number of bytes in UTF-8 encoding in each - component of a path. A value of 0 will disable the check. + component of a path. A value of 0 will disable the check. Support + multiple size unit suffix(case insensitive), as described in dfs.blocksize. + @@ -406,8 +409,9 @@ 1048576 Minimum block size in bytes, enforced by the Namenode at create time. This prevents the accidental creation of files with tiny block - sizes (and thus many blocks), which can degrade - performance. + sizes (and thus many blocks), which can degrade performance. Support multiple + size unit suffix(case insensitive), as described in dfs.blocksize. + @@ -945,7 +949,8 @@ 104857600 The amount of space to reserve/require for a NameNode storage directory - in bytes. The default is 100MB. + in bytes. The default is 100MB. Support multiple size unit + suffix(case insensitive), as described in dfs.blocksize. @@ -1331,6 +1336,8 @@ A default value of 0 indicates that throttling is disabled. The maximum bandwidth used for bootstrapping standby namenode is configured with dfs.image.transfer-bootstrap-standby.bandwidthPerSec. + Support multiple size unit suffix(case insensitive), as described + in dfs.blocksize. @@ -1344,6 +1351,8 @@ value should be used in most cases, to ensure timely HA operations. The maximum bandwidth used for regular image transfers is configured with dfs.image.transfer.bandwidthPerSec. + Support multiple size unit suffix(case insensitive), as described in + dfs.blocksize. @@ -1354,6 +1363,8 @@ Chunksize in bytes to upload the checkpoint. Chunked streaming is used to avoid internal buffering of contents of image file of huge size. + Support multiple size unit suffix(case insensitive), as described + in dfs.blocksize. @@ -2293,7 +2304,8 @@ bytes of free disk space before they are considered imbalanced. If the free space of all the volumes are within this range of each other, the volumes will be considered balanced and block assignments will be done on a pure - round robin basis. + round robin basis. Support multiple size unit suffix(case insensitive), as + described in dfs.blocksize. @@ -2366,7 +2378,8 @@ read ahead in the block file using posix_fadvise, potentially decreasing I/O wait times. Unlike dfs.datanode.readahead.bytes, this is a client-side setting rather than a setting for the entire datanode. If present, this - setting will override the DataNode default. + setting will override the DataNode default. Support multiple size unit + suffix(case insensitive), as described in dfs.blocksize. When using local reads, this setting determines how much readahead we do in BlockReaderLocal. @@ -2510,7 +2523,8 @@ The amount of memory in bytes to use for caching of block replicas in memory on the datanode. The datanode's maximum locked memory soft ulimit (RLIMIT_MEMLOCK) must be set to at least this value, else the datanode - will abort on startup. + will abort on startup. Support multiple size unit suffix(case insensitive), + as described in dfs.blocksize. By default, this parameter is set to 0, which disables in-memory caching. @@ -2835,6 +2849,8 @@ The maximum combined size of the name and value of an extended attribute in bytes. It should be larger than 0, and less than or equal to maximum size hard limit which is 32768. + Support multiple size unit suffix(case insensitive), as described in + dfs.blocksize.