big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "HDFS-12914. Block report leases cause missing blocks until next report. Contributed by Santosh Marella, He Xiaoqiao."

Browse Source 
This reverts commit ae4143a529.
This commit is contained in:
Wei-Chiu Chuang 2019-06-17 16:18:48 -07:00
parent a50c35bb81
commit 7314185c4a
2 changed files with 21 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -2572,21 +2572,6 @@ private static class BlockInfoToAdd {
} }
} }
/**
* Check block report lease.
* @return true if lease exist and not expire
*/
public boolean checkBlockReportLease(BlockReportContext context,
final DatanodeID nodeID) throws UnregisteredNodeException {
if (context == null) {
return true;
}
DatanodeDescriptor node = datanodeManager.getDatanode(nodeID);
final long startTime = Time.monotonicNow();
return blockReportLeaseManager.checkLease(node, startTime,
context.getLeaseId());
}
/** /**
* The given storage is reporting all its blocks. * The given storage is reporting all its blocks.
* Update the (storage{@literal -->}block list) and * Update the (storage{@literal -->}block list) and
@ -2634,6 +2619,12 @@ public boolean processReport(final DatanodeID nodeID,
blockReportLeaseManager.removeLease(node); blockReportLeaseManager.removeLease(node);
return !node.hasStaleStorages(); return !node.hasStaleStorages();
} }
if (context != null) {
if (!blockReportLeaseManager.checkLease(node, startTime,
context.getLeaseId())) {
return false;
}
}
if (storageInfo.getBlockReportCount() == 0) { if (storageInfo.getBlockReportCount() == 0) {
// The first block report can be processed a lot more efficiently than // The first block report can be processed a lot more efficiently than

34
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
View File

@ -45,6 +45,7 @@
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.Callable;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
@ -174,7 +175,6 @@
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.NodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
@ -1591,25 +1591,21 @@ public DatanodeCommand blockReport(final DatanodeRegistration nodeReg,
} }
final BlockManager bm = namesystem.getBlockManager(); final BlockManager bm = namesystem.getBlockManager();
boolean noStaleStorages = false; boolean noStaleStorages = false;
try { for (int r = 0; r < reports.length; r++) {
if (bm.checkBlockReportLease(context, nodeReg)) { final BlockListAsLongs blocks = reports[r].getBlocks();
for (int r = 0; r < reports.length; r++) { //
final BlockListAsLongs blocks = reports[r].getBlocks(); // BlockManager.processReport accumulates information of prior calls
// // for the same node and storage, so the value returned by the last
// BlockManager.processReport accumulates information of prior calls // call of this loop is the final updated value for noStaleStorage.
// for the same node and storage, so the value returned by the last //
// call of this loop is the final updated value for noStaleStorage. final int index = r;
// noStaleStorages = bm.runBlockOp(new Callable<Boolean>() {
final int index = r; @Override
noStaleStorages = bm.runBlockOp(() -> public Boolean call() throws IOException {
bm.processReport(nodeReg, reports[index].getStorage(), return bm.processReport(nodeReg, reports[index].getStorage(),
blocks, context)); blocks, context);
} }
} });
} catch (UnregisteredNodeException une) {
LOG.debug("Datanode {} is attempting to report but not register yet.",
nodeReg);
return RegisterCommand.REGISTER;
} }
bm.removeBRLeaseIfNeeded(nodeReg, context); bm.removeBRLeaseIfNeeded(nodeReg, context);