big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-2759. Pre-allocate HDFS edit log files after writing version number. Contributed by Aaron T. Myers.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1237153 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Aaron Myers 2012-01-28 20:26:13 +00:00
parent 934047630a
commit 976c07696d
3 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -172,6 +172,9 @@ Trunk (unreleased changes)
HDFS-2768. BackupNode stop can not close proxy connections because HDFS-2768. BackupNode stop can not close proxy connections because
it is not a proxy instance. (Uma Maheswara Rao G via eli) it is not a proxy instance. (Uma Maheswara Rao G via eli)
HDFS-2759. Pre-allocate HDFS edit log files after writing version number.
(atm)
Release 0.23.1 - UNRELEASED Release 0.23.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
View File

@ -167,10 +167,10 @@ protected void flushAndSync() throws IOException {
LOG.info("Nothing to flush"); LOG.info("Nothing to flush");
return; return;
} }
preallocate(); // preallocate file if necessary
doubleBuf.flushTo(fp); doubleBuf.flushTo(fp);
fc.force(false); // metadata updates not needed because of preallocation fc.force(false); // metadata updates not needed
fc.position(fc.position() - 1); // skip back the end-of-file marker fc.position(fc.position() - 1); // skip back the end-of-file marker
preallocate(); // preallocate file if necessary
} }
/** /**

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
View File

@ -41,6 +41,7 @@
public class TestEditLogFileOutputStream { public class TestEditLogFileOutputStream {
private final static long PREALLOCATION_LENGTH = (1024 * 1024) + 4;
private final static int HEADER_LEN = 17; private final static int HEADER_LEN = 17;
private static final File TEST_EDITS = private static final File TEST_EDITS =
new File(System.getProperty("test.build.data","/tmp"), new File(System.getProperty("test.build.data","/tmp"),
@ -65,8 +66,9 @@ public void testPreallocation() throws IOException {
assertEquals("Edit log should contain a header as valid length", assertEquals("Edit log should contain a header as valid length",
HEADER_LEN, validation.getValidLength()); HEADER_LEN, validation.getValidLength());
assertEquals(1, validation.getNumTransactions()); assertEquals(1, validation.getNumTransactions());
assertEquals("Edit log should have 1MB of bytes allocated", assertEquals("Edit log should have 1MB pre-allocated, plus 4 bytes " +
1024*1024, editLog.length()); "for the version number",
PREALLOCATION_LENGTH, editLog.length());
cluster.getFileSystem().mkdirs(new Path("/tmp"), cluster.getFileSystem().mkdirs(new Path("/tmp"),
@ -79,8 +81,8 @@ public void testPreallocation() throws IOException {
validation.getValidLength() > oldLength); validation.getValidLength() > oldLength);
assertEquals(2, validation.getNumTransactions()); assertEquals(2, validation.getNumTransactions());
assertEquals("Edit log should be 1MB long", assertEquals("Edit log should be 1MB long, plus 4 bytes for the version number",
1024 * 1024, editLog.length()); PREALLOCATION_LENGTH, editLog.length());
// 256 blocks for the 1MB of preallocation space // 256 blocks for the 1MB of preallocation space
assertTrue("Edit log disk space used should be at least 257 blocks", assertTrue("Edit log disk space used should be at least 257 blocks",
256 * 4096 <= new DU(editLog, conf).getUsed()); 256 * 4096 <= new DU(editLog, conf).getUsed());