From 1e3e246934b274d28bbfebdb5306e7a71b555be9 Mon Sep 17 00:00:00 2001 From: WangYuanben <48795318+YuanbenWang@users.noreply.github.com> Date: Tue, 8 Aug 2023 07:37:26 +0800 Subject: [PATCH] HADOOP-18810. Document missing a lot of properties in core-default.xml. (#5912) Contributed by WangYuanben. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../src/main/resources/core-default.xml | 316 +++++++++++++++++- .../conf/TestCommonConfigurationFields.java | 1 + 2 files changed, 315 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 5f841bd233..14ffe3d9de 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -73,6 +73,27 @@ Is service-level authorization enabled? + + security.service.authorization.default.acl + + + Define the default acl for the Hadoop service if the acl of Hadoop + service is not defined in hadoop-policy.xml. If not set, `*` is applied + meaning that all users are allowed to access the service. The list of + users and groups are both comma-separated list of names separated by + a space. Example: `user1,user2 group1,group2`. + + + + + security.service.authorization.default.acl.blocked + + + This property specifies the list of users and groups who are not + authorized to access Hadoop service. + + + hadoop.security.instrumentation.requires.admin false @@ -225,6 +246,17 @@ + + hadoop.security.group.mapping.ldap.ctx.factory.class + + + Used to specify the fully qualified class name of the initial context + factory when connecting to an LDAP server. The default value is + "com.sun.jndi.ldap.LdapCtxFactory", but set to null now to avoid + LifecycleExecutionException with JDK 11(see HADOOP-15941). + + + hadoop.security.group.mapping.ldap.connection.timeout.ms 60000 @@ -803,7 +835,19 @@ hadoop.token.files - List of token cache files that have delegation tokens for hadoop service + + A comma-separated list of token cache files that have delegation tokens + for hadoop service + + + + + hadoop.tokens + + + A comma-separated list of delegation tokens from base64 encoding + for hadoop service. + @@ -855,6 +899,65 @@ operate entirely in Java, specify "java-builtin". + + io.compression.codec.lz4.buffersize + 262144 + + Internal buffer size for Lz4 compressor/decompressors. + + + + + io.compression.codec.lz4.use.lz4hc + false + + Enable lz4hc(slow but with high compression ratio) for lz4 compression. + + + + + io.compression.codec.lzo.buffersize + 65536 + + Internal buffer size for Lzo compressor/decompressors. + + + + + io.compression.codec.lzo.class + org.apache.hadoop.io.compress.LzoCodec + + Codec class that implements Lzo compression algorithm. + + + + + io.compression.codec.snappy.buffersize + 262144 + + Internal buffer size for Snappy compressor/decompressors. + + + + + io.compression.codec.zstd.buffersize + 0 + + Indicate ZStandard buffer size. The default value 0 means use the + recommended zstd buffer size that the library recommends. + + + + + io.compression.codec.zstd.level + 3 + + Indicate ZStandard compression level. The higher the compression level, + the higher the compression ratio and memory usage, but the slower the + compression and decompression speed. + + + io.serializations org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization @@ -1145,6 +1248,33 @@ + + fs.file.impl + + + Specify the implementation class used for accessing the file system. It + is a fully qualified class name, including both the package name and the + class name. + + + + + fs.creation.parallel.count + 64 + + This property sets a a semaphore to throttle the number of FileSystem + instances which can be created simultaneously. This is designed to reduce + the impact of many threads in an application calling FileSystem#get() on + a filesystem which takes time to instantiate -for example to an object + where HTTPS connections are set up during initialization. Many threads + trying to do this may create spurious delays by conflicting for access + to synchronized blocks, when simply limiting the parallelism diminishes + the conflict, so speeds up all threads trying to access the store. If a + service appears to be blocking on all threads initializing connections to + abfs, s3a or store, try a smaller (possibly significantly smaller) value. + + + fs.AbstractFileSystem.ftp.impl org.apache.hadoop.fs.ftp.FtpFs @@ -1231,6 +1361,22 @@ + + fs.iostatistics.logging.level + debug + + Logging level for IOStatistics. + + + + + fs.iostatistics.thread.level.enabled + true + + Enable IOStatisticsContext support for thread level. + + + fs.s3a.access.key AWS access key ID used by S3A file system. Omit for IAM role-based or provider-based authentication. @@ -2230,6 +2376,13 @@ The switch to turn S3A auditing on or off. + + ipc.client.async.calls.max + 100 + + Define the maximum number of outstanding async calls. + + ipc.client.idlethreshold @@ -2239,6 +2392,14 @@ The switch to turn S3A auditing on or off. + + ipc.client.connection.idle-scan-interval.ms + 10000 + + Indicate how often the server scans for idle connections. + + + ipc.client.kill.max 10 @@ -2286,6 +2447,14 @@ The switch to turn S3A auditing on or off. + + ipc.client.connect.max.retries.on.sasl + 5 + + The maximum retries on SASL connection failures in RPC client. + + + ipc.client.tcpnodelay true @@ -2329,6 +2498,14 @@ The switch to turn S3A auditing on or off. + + ipc.server.tcpnodelay + true + + If true then disable Nagle's Algorithm. + + + ipc.server.handler.queue.size 100 @@ -2338,6 +2515,24 @@ The switch to turn S3A auditing on or off. + + ipc.server.max.response.size + 1048576 + + The maximum size when large IPC handler response buffer is reset. + + + + + ipc.server.metrics.update.runner.interval + 5000 + + To configure scheduling of server metrics update thread. This config is + used to indicate initial delay and delay between each execution of the + metric update runnable thread. + + + ipc.server.listen.queue.size 256 @@ -2363,6 +2558,22 @@ The switch to turn S3A auditing on or off. + + ipc.server.read.connection-queue.size + 100 + + Number of pending connections that may be queued per socket reader. + + + + + ipc.server.read.threadpool.size + 1 + + Indicates the number of threads in RPC server reading from the socket. + + + ipc.maximum.data.length 134217728 @@ -2392,6 +2603,14 @@ The switch to turn S3A auditing on or off. + + callqueue.overflow.trigger.failover + false + + Enable callqueue overflow trigger failover for stateless servers. + + + @@ -2476,6 +2695,20 @@ The switch to turn S3A auditing on or off. + + ipc.[port_number].callqueue.capacity.weights + + + When FairCallQueue is enabled, user can specify capacity allocation + among all sub-queues via this property. The value of this config is + a comma-separated list of positive integers, each of which specifies + the weight associated with the sub-queue at that index. This list + length should be IPC scheduler priority levels, defined by + "scheduler.priority.levels". By default, each sub-queue is associated + with weight 1, i.e., all sub-queues are allocated with the same capacity. + + + ipc.[port_number].scheduler.priority.levels 4 @@ -2744,6 +2977,24 @@ The switch to turn S3A auditing on or off. + + net.topology.configured.node.mapping + + + Key to define the node mapping as a comma-delimited list of host=rack + mappings. e.g. host1=r1,host2=r1,host3=r2. Important: spaces not trimmed + and are considered significant. + + + + + net.topology.dependency.script.file.name + + + Key to the dependency script filename. + + + file.stream-buffer-size @@ -3274,6 +3525,17 @@ The switch to turn S3A auditing on or off. + + hadoop.user.group.metrics.percentiles.intervals + + + A comma-delimited list of integers denoting the desired rollover + intervals (in seconds) for percentile latency metrics on the Namenode + and Datanode for each user in the group. By default, percentile + latency metrics are disabled. + + + rpc.metrics.quantile.enable false @@ -3532,6 +3794,24 @@ The switch to turn S3A auditing on or off. + + hadoop.security.kms.client.failover.max.retries + + + Default value is the number of providers specified. + + + + + hadoop.security.kerberos.ticket.cache.path + + + Path to the Kerberos ticket cache. Setting this will force + UserGroupInformation to use only this ticket cache file when + creating a FileSystem instance. + + + ipc.server.max.connections 0 @@ -3944,6 +4224,30 @@ The switch to turn S3A auditing on or off. + + hadoop.zk.server.principal + + + Principal name for zookeeper servers. + + + + + hadoop.zk.kerberos.principal + + + Kerberos principal name for zookeeper connection. + + + + + hadoop.zk.kerberos.keytab + + + Kerberos keytab for zookeeper connection. + + + hadoop.zk.ssl.keystore.location @@ -3986,7 +4290,15 @@ The switch to turn S3A auditing on or off. YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL - System tags to group related properties together. + A comma-separated list of system tags to group related properties together. + + + + + hadoop.tags.custom + + + A comma-separated list of custom tags to group related properties together. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index 74b2f55065..f7303fb0f5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -157,6 +157,7 @@ public void initializeMemberVariables() { xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.impl"); xmlPropsToSkipCompare.add("ipc.scheduler.impl"); xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.priority.levels"); + xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.capacity.weights"); xmlPropsToSkipCompare.add( "ipc.[port_number].faircallqueue.multiplexer.weights"); xmlPropsToSkipCompare.add("ipc.[port_number].identity-provider.impl");