big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16732. [SBN READ] Avoid get location from observer when the block report is delayed (#4756)

Browse Source 
Signed-off-by: Erik Krogen <xkrogen@apache.org>
This commit is contained in:
zhengchenyu 2022-08-26 01:37:25 +08:00 committed by GitHub
parent 7fb9c306e2
commit 231a4468cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 34 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -103,6 +103,7 @@
import org.apache.commons.text.CaseUtils; import org.apache.commons.text.CaseUtils;
import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult;
import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*;
@ -2202,14 +2203,8 @@ LocatedBlocks getBlockLocations(String clientMachine, String srcArg,
} }
} }
} }
} else if (haEnabled && haContext != null && } else if (isObserver()) {
haContext.getState().getServiceState() == OBSERVER) { checkBlockLocationsWhenObserver(res.blocks, srcArg);
for (LocatedBlock b : res.blocks.getLocatedBlocks()) {
if (b.getLocations() == null || b.getLocations().length == 0) {
throw new ObserverRetryOnActiveException("Zero blocklocations "
+ "for " + srcArg);
}
}
} }
} finally { } finally {
readUnlock(operationName, getLockReportInfoSupplier(srcArg)); readUnlock(operationName, getLockReportInfoSupplier(srcArg));
@ -3470,6 +3465,10 @@ HdfsFileStatus getFileInfo(final String src, boolean resolveLink,
logAuditEvent(false, operationName, src); logAuditEvent(false, operationName, src);
throw e; throw e;
} }
if (needLocation && isObserver() && stat instanceof HdfsLocatedFileStatus) {
LocatedBlocks lbs = ((HdfsLocatedFileStatus) stat).getLocatedBlocks();
checkBlockLocationsWhenObserver(lbs, src);
}
logAuditEvent(true, operationName, src); logAuditEvent(true, operationName, src);
return stat; return stat;
} }
@ -4175,6 +4174,14 @@ DirectoryListing getListing(String src, byte[] startAfter,
logAuditEvent(false, operationName, src); logAuditEvent(false, operationName, src);
throw e; throw e;
} }
if (needLocation && isObserver()) {
for (HdfsFileStatus fs : dl.getPartialListing()) {
if (fs instanceof HdfsLocatedFileStatus) {
LocatedBlocks lbs = ((HdfsLocatedFileStatus) fs).getLocatedBlocks();
checkBlockLocationsWhenObserver(lbs, fs.toString());
}
}
}
logAuditEvent(true, operationName, src); logAuditEvent(true, operationName, src);
return dl; return dl;
} }
@ -9020,4 +9027,17 @@ public void checkErasureCodingSupported(String operationName)
throw new UnsupportedActionException(operationName + " not supported."); throw new UnsupportedActionException(operationName + " not supported.");
} }
} }
private boolean isObserver() {
return haEnabled && haContext != null && haContext.getState().getServiceState() == OBSERVER;
}
private void checkBlockLocationsWhenObserver(LocatedBlocks blocks, String src)
throws ObserverRetryOnActiveException {
for (LocatedBlock b : blocks.getLocatedBlocks()) {
if (b.getLocations() == null || b.getLocations().length == 0) {
throw new ObserverRetryOnActiveException("Zero blocklocations for " + src);
}
}
}
} }

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
View File

@ -369,6 +369,12 @@ public void testObserverNodeBlockMissingRetry() throws Exception {
dfs.open(testPath); dfs.open(testPath);
assertSentTo(0); assertSentTo(0);
dfs.getClient().listPaths("/", new byte[0], true);
assertSentTo(0);
dfs.getClient().getLocatedFileInfo(testPath.toString(), false);
assertSentTo(0);
Mockito.reset(bmSpy); Mockito.reset(bmSpy);
} }