big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14440. RBF: Optimize the file write process in case of multiple destinations. Contributed by Ayush Saxena.

Browse Source
This commit is contained in:
Ayush Saxena 2019-05-24 20:37:58 +05:30 committed by Brahma Reddy Battula
parent 2636a54ffd
commit 8e4267650f
1 changed files with 25 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
View File

@ -631,28 +631,11 @@ RemoteLocation getCreateLocation(
RemoteLocation createLocation = locations.get(0); RemoteLocation createLocation = locations.get(0);
if (locations.size() > 1) { if (locations.size() > 1) {
try { try {
// Check if this file already exists in other subclusters RemoteLocation existingLocation = getExistingLocation(src, locations);
LocatedBlocks existingLocation = getBlockLocations(src, 0, 1); // Forward to the existing location and let the NN handle the error
if (existingLocation != null) { if (existingLocation != null) {
// Forward to the existing location and let the NN handle the error LOG.debug("{} already exists in {}.", src, existingLocation);
LocatedBlock existingLocationLastLocatedBlock = createLocation = existingLocation;
existingLocation.getLastLocatedBlock();
if (existingLocationLastLocatedBlock == null) {
// The block has no blocks yet, check for the meta data
for (RemoteLocation location : locations) {
RemoteMethod method = new RemoteMethod("getFileInfo",
new Class<?>[] {String.class}, new RemoteParam());
if (rpcClient.invokeSingle(location, method) != null) {
createLocation = location;
break;
}
}
} else {
ExtendedBlock existingLocationLastBlock =
existingLocationLastLocatedBlock.getBlock();
String blockPoolId = existingLocationLastBlock.getBlockPoolId();
createLocation = getLocationForPath(src, true, blockPoolId);
}
} }
} catch (FileNotFoundException fne) { } catch (FileNotFoundException fne) {
// Ignore if the file is not found // Ignore if the file is not found
@ -661,6 +644,27 @@ RemoteLocation getCreateLocation(
return createLocation; return createLocation;
} }
/**
* Gets the remote location where the file exists.
* @param src the name of file.
* @param locations all the remote locations.
* @return the remote location of the file if it exists, else null.
* @throws IOException in case of any exception.
*/
private RemoteLocation getExistingLocation(String src,
List<RemoteLocation> locations) throws IOException {
RemoteMethod method = new RemoteMethod("getFileInfo",
new Class<?>[] {String.class}, new RemoteParam());
Map<RemoteLocation, HdfsFileStatus> results = rpcClient.invokeConcurrent(
locations, method, false, false, HdfsFileStatus.class);
for (RemoteLocation loc : locations) {
if (results.get(loc) != null) {
return loc;
}
}
return null;
}
@Override // ClientProtocol @Override // ClientProtocol
public LastBlockWithStatus append(String src, final String clientName, public LastBlockWithStatus append(String src, final String clientName,
final EnumSetWritable<CreateFlag> flag) throws IOException { final EnumSetWritable<CreateFlag> flag) throws IOException {