big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11345. Document the configuration key for FSNamesystem lock fairness. Contributed by Erik Krogen.

Browse Source
This commit is contained in:
Akira Ajisaka 2017-06-20 16:35:54 +09:00
parent 15964ef147
commit 2c769167db
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
4 changed files with 22 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -423,6 +423,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long public static final long
DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT = 25; DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT = 25;
public static final String DFS_NAMENODE_FSLOCK_FAIR_KEY =
"dfs.namenode.fslock.fair";
public static final boolean DFS_NAMENODE_FSLOCK_FAIR_DEFAULT = true;
public static final String DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY = public static final String DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY =
"dfs.namenode.lock.detailed-metrics.enabled"; "dfs.namenode.lock.detailed-metrics.enabled";
public static final boolean DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT = public static final boolean DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT =

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
View File

@ -32,6 +32,8 @@
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT;
@ -113,7 +115,8 @@ public Long initialValue() {
@VisibleForTesting @VisibleForTesting
FSNamesystemLock(Configuration conf, FSNamesystemLock(Configuration conf,
MutableRatesWithAggregation detailedHoldTimeMetrics, Timer timer) { MutableRatesWithAggregation detailedHoldTimeMetrics, Timer timer) {
boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true); boolean fair = conf.getBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY,
DFS_NAMENODE_FSLOCK_FAIR_DEFAULT);
FSNamesystem.LOG.info("fsLock is fair: " + fair); FSNamesystem.LOG.info("fsLock is fair: " + fair);
this.coarseLock = new ReentrantReadWriteLock(fair); this.coarseLock = new ReentrantReadWriteLock(fair);
this.timer = timer; this.timer = timer;

10
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -2759,6 +2759,16 @@
</description> </description>
</property> </property>
<property>
<name>dfs.namenode.fslock.fair</name>
<value>true</value>
<description>If this is true, the FS Namesystem lock will be used in Fair mode,
which will help to prevent writer threads from being starved, but can provide
lower lock throughput. See java.util.concurrent.locks.ReentrantReadWriteLock
for more information on fair/non-fair locks.
</description>
</property>
<property> <property>
<name>dfs.namenode.startup.delay.block.deletion.sec</name> <name>dfs.namenode.startup.delay.block.deletion.sec</name>
<value>0</value> <value>0</value>

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
View File

@ -40,6 +40,7 @@
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_KEY;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter; import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.assertGauge; import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
@ -53,11 +54,11 @@ public class TestFSNamesystemLock {
public void testFsLockFairness() throws IOException, InterruptedException{ public void testFsLockFairness() throws IOException, InterruptedException{
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.setBoolean("dfs.namenode.fslock.fair", true); conf.setBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY, true);
FSNamesystemLock fsnLock = new FSNamesystemLock(conf, null); FSNamesystemLock fsnLock = new FSNamesystemLock(conf, null);
assertTrue(fsnLock.coarseLock.isFair()); assertTrue(fsnLock.coarseLock.isFair());
conf.setBoolean("dfs.namenode.fslock.fair", false); conf.setBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY, false);
fsnLock = new FSNamesystemLock(conf, null); fsnLock = new FSNamesystemLock(conf, null);
assertFalse(fsnLock.coarseLock.isFair()); assertFalse(fsnLock.coarseLock.isFair());
} }
@ -103,7 +104,7 @@ public void testFSLockGetWaiterCount() throws InterruptedException {
final int threadCount = 3; final int threadCount = 3;
final CountDownLatch latch = new CountDownLatch(threadCount); final CountDownLatch latch = new CountDownLatch(threadCount);
final Configuration conf = new Configuration(); final Configuration conf = new Configuration();
conf.setBoolean("dfs.namenode.fslock.fair", true); conf.setBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY, true);
final FSNamesystemLock rwLock = new FSNamesystemLock(conf, null); final FSNamesystemLock rwLock = new FSNamesystemLock(conf, null);
rwLock.writeLock(); rwLock.writeLock();
ExecutorService helper = Executors.newFixedThreadPool(threadCount); ExecutorService helper = Executors.newFixedThreadPool(threadCount);