big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-15167. Block Report Interval shouldn't be reset apart from first Block Report. Contributed by Ayush Saxena.

Browse Source
This commit is contained in:
Surendra Singh Lilhore 2020-02-27 14:08:42 +05:30
parent 7dfa37e8f0
commit 2059f255d3
3 changed files with 26 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
View File

@ -496,7 +496,7 @@ class BPOfferService {
*/ */
void scheduleBlockReport(long delay) { void scheduleBlockReport(long delay) {
for (BPServiceActor actor : bpServices) { for (BPServiceActor actor : bpServices) {
actor.getScheduler().scheduleBlockReport(delay); actor.getScheduler().scheduleBlockReport(delay, false);
} }
} }

30
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
View File

@ -828,7 +828,7 @@ class BPServiceActor implements Runnable {
fullBlockReportLeaseId = 0; fullBlockReportLeaseId = 0;
// random short delay - helps scatter the BR from all DNs // random short delay - helps scatter the BR from all DNs
scheduler.scheduleBlockReport(dnConf.initialBlockReportDelayMs); scheduler.scheduleBlockReport(dnConf.initialBlockReportDelayMs, true);
} }
@ -1217,14 +1217,19 @@ class BPServiceActor implements Runnable {
void forceFullBlockReportNow() { void forceFullBlockReportNow() {
forceFullBlockReport.set(true); forceFullBlockReport.set(true);
resetBlockReportTime = true;
} }
/** /**
* This methods arranges for the data node to send the block report at * This methods arranges for the data node to send the block report at
* the next heartbeat. * the next heartbeat.
* @param delay specifies the maximum amount of random delay(in
* milliseconds) in sending the block report. A value of 0
* or less makes the BR to go right away without any delay.
* @param isRegistration if true, resets the future BRs for randomness,
* post first BR to avoid regular BRs from all DN's
* coming at one time.
*/ */
long scheduleBlockReport(long delay) { long scheduleBlockReport(long delay, boolean isRegistration) {
if (delay > 0) { // send BR after random delay if (delay > 0) { // send BR after random delay
// Numerical overflow is possible here and is okay. // Numerical overflow is possible here and is okay.
nextBlockReportTime = nextBlockReportTime =
@ -1232,7 +1237,9 @@ class BPServiceActor implements Runnable {
} else { // send at next heartbeat } else { // send at next heartbeat
nextBlockReportTime = monotonicNow(); nextBlockReportTime = monotonicNow();
} }
resetBlockReportTime = true; // reset future BRs for randomness resetBlockReportTime = isRegistration; // reset future BRs for
// randomness, post first block report to avoid regular BRs from all
// DN's coming at one time.
return nextBlockReportTime; return nextBlockReportTime;
} }
@ -1257,9 +1264,18 @@ class BPServiceActor implements Runnable {
* 2) unexpected like 21:35:43, next report should be at 2:20:14 * 2) unexpected like 21:35:43, next report should be at 2:20:14
* on the next day. * on the next day.
*/ */
nextBlockReportTime += long factor =
(((monotonicNow() - nextBlockReportTime + blockReportIntervalMs) / (monotonicNow() - nextBlockReportTime + blockReportIntervalMs)
blockReportIntervalMs)) * blockReportIntervalMs; / blockReportIntervalMs;
if (factor != 0) {
nextBlockReportTime += factor * blockReportIntervalMs;
} else {
// If the difference between the present time and the scheduled
// time is very less, the factor can be 0, so in that case, we can
// ignore that negligible time, spent while sending the BRss and
// schedule the next BR after the blockReportInterval.
nextBlockReportTime += blockReportIntervalMs;
}
} }
} }

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
View File

@ -68,7 +68,7 @@ public class TestBpServiceActorScheduler {
public void testScheduleBlockReportImmediate() { public void testScheduleBlockReportImmediate() {
for (final long now : getTimestamps()) { for (final long now : getTimestamps()) {
Scheduler scheduler = makeMockScheduler(now); Scheduler scheduler = makeMockScheduler(now);
scheduler.scheduleBlockReport(0); scheduler.scheduleBlockReport(0, true);
assertTrue(scheduler.resetBlockReportTime); assertTrue(scheduler.resetBlockReportTime);
assertThat(scheduler.nextBlockReportTime, is(now)); assertThat(scheduler.nextBlockReportTime, is(now));
} }
@ -79,7 +79,7 @@ public class TestBpServiceActorScheduler {
for (final long now : getTimestamps()) { for (final long now : getTimestamps()) {
Scheduler scheduler = makeMockScheduler(now); Scheduler scheduler = makeMockScheduler(now);
final long delayMs = 10; final long delayMs = 10;
scheduler.scheduleBlockReport(delayMs); scheduler.scheduleBlockReport(delayMs, true);
assertTrue(scheduler.resetBlockReportTime); assertTrue(scheduler.resetBlockReportTime);
assertTrue(scheduler.nextBlockReportTime - now >= 0); assertTrue(scheduler.nextBlockReportTime - now >= 0);
assertTrue(scheduler.nextBlockReportTime - (now + delayMs) < 0); assertTrue(scheduler.nextBlockReportTime - (now + delayMs) < 0);