HDFS-3020. Fix editlog to automatically sync when buffer is full. Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1295239 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
ca9b4de772
commit
28de3628f5
@ -253,6 +253,8 @@ Release 0.23.3 - UNRELEASED
|
||||
HDFS-2968. Protocol translator for BlockRecoveryCommand broken when
|
||||
multiple blocks need recovery. (todd)
|
||||
|
||||
HDFS-3020. Fix editlog to automatically sync when buffer is full. (todd)
|
||||
|
||||
Release 0.23.2 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
@ -86,7 +86,7 @@ void flushTo(OutputStream out) throws IOException {
|
||||
}
|
||||
|
||||
boolean shouldForceSync() {
|
||||
return bufReady.size() >= initBufferSize;
|
||||
return bufCurrent.size() >= initBufferSize;
|
||||
}
|
||||
|
||||
DataOutputBuffer getCurrentBuf() {
|
||||
|
@ -735,6 +735,14 @@ List<JournalAndStream> getJournals() {
|
||||
synchronized void setRuntimeForTesting(Runtime runtime) {
|
||||
this.runtime = runtime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Used only by tests.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
void setMetricsForTests(NameNodeMetrics metrics) {
|
||||
this.metrics = metrics;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a manifest of what finalized edit logs are available
|
||||
|
@ -50,6 +50,7 @@
|
||||
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
|
||||
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
|
||||
import org.apache.hadoop.test.GenericTestUtils;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.log4j.Level;
|
||||
@ -798,6 +799,40 @@ public void testFailedOpen() throws Exception {
|
||||
log.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Regression test for HDFS-1112/HDFS-3020. Ensures that, even if
|
||||
* logSync isn't called periodically, the edit log will sync itself.
|
||||
*/
|
||||
public void testAutoSync() throws Exception {
|
||||
File logDir = new File(TEST_DIR, "testAutoSync");
|
||||
logDir.mkdirs();
|
||||
FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir);
|
||||
|
||||
String oneKB = StringUtils.byteToHexString(
|
||||
new byte[500]);
|
||||
|
||||
try {
|
||||
log.open();
|
||||
NameNodeMetrics mockMetrics = Mockito.mock(NameNodeMetrics.class);
|
||||
log.setMetricsForTests(mockMetrics);
|
||||
|
||||
for (int i = 0; i < 400; i++) {
|
||||
log.logDelete(oneKB, 1L);
|
||||
}
|
||||
// After ~400KB, we're still within the 512KB buffer size
|
||||
Mockito.verify(mockMetrics, Mockito.times(0)).addSync(Mockito.anyLong());
|
||||
|
||||
// After ~400KB more, we should have done an automatic sync
|
||||
for (int i = 0; i < 400; i++) {
|
||||
log.logDelete(oneKB, 1L);
|
||||
}
|
||||
Mockito.verify(mockMetrics, Mockito.times(1)).addSync(Mockito.anyLong());
|
||||
|
||||
} finally {
|
||||
log.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests the getEditLogManifest function using mock storage for a number
|
||||
|
Loading…
Reference in New Issue
Block a user