diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a6a67018f0..999c32f214 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -253,6 +253,8 @@ Release 0.23.3 - UNRELEASED HDFS-2968. Protocol translator for BlockRecoveryCommand broken when multiple blocks need recovery. (todd) + HDFS-3020. Fix editlog to automatically sync when buffer is full. (todd) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java index 5312b145ae..2aa736d638 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java @@ -86,7 +86,7 @@ void flushTo(OutputStream out) throws IOException { } boolean shouldForceSync() { - return bufReady.size() >= initBufferSize; + return bufCurrent.size() >= initBufferSize; } DataOutputBuffer getCurrentBuf() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index 7a68429bdf..80c608814a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -735,6 +735,14 @@ List getJournals() { synchronized void setRuntimeForTesting(Runtime runtime) { this.runtime = runtime; } + + /** + * Used only by tests. + */ + @VisibleForTesting + void setMetricsForTests(NameNodeMetrics metrics) { + this.metrics = metrics; + } /** * Return a manifest of what finalized edit logs are available diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java index cf56c02d7a..b3eeeab41d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.NNStorage; +import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; @@ -798,6 +799,40 @@ public void testFailedOpen() throws Exception { log.close(); } } + + /** + * Regression test for HDFS-1112/HDFS-3020. Ensures that, even if + * logSync isn't called periodically, the edit log will sync itself. + */ + public void testAutoSync() throws Exception { + File logDir = new File(TEST_DIR, "testAutoSync"); + logDir.mkdirs(); + FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir); + + String oneKB = StringUtils.byteToHexString( + new byte[500]); + + try { + log.open(); + NameNodeMetrics mockMetrics = Mockito.mock(NameNodeMetrics.class); + log.setMetricsForTests(mockMetrics); + + for (int i = 0; i < 400; i++) { + log.logDelete(oneKB, 1L); + } + // After ~400KB, we're still within the 512KB buffer size + Mockito.verify(mockMetrics, Mockito.times(0)).addSync(Mockito.anyLong()); + + // After ~400KB more, we should have done an automatic sync + for (int i = 0; i < 400; i++) { + log.logDelete(oneKB, 1L); + } + Mockito.verify(mockMetrics, Mockito.times(1)).addSync(Mockito.anyLong()); + + } finally { + log.close(); + } + } /** * Tests the getEditLogManifest function using mock storage for a number