big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15937. Reduce memory used during datanode layout upgrade. Contributed by Stephen O'Donnell (#2838)

Browse Source 
(cherry picked from commit 4c567fcff7)
This commit is contained in:
Stephen O'Donnell 2021-04-08 11:59:02 +01:00 committed by S O'Donnell
parent dca2bf9dd5
commit ef95f7a963
1 changed files with 62 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
View File

@ -1064,12 +1064,26 @@ private static void linkAllBlocks(File fromDir, File fromBbwDir, File toDir,
} }
private static class LinkArgs { private static class LinkArgs {
File src; private File srcDir;
File dst; private File dstDir;
private String blockFile;
LinkArgs(File src, File dst) { LinkArgs(File srcDir, File dstDir, String blockFile) {
this.src = src; this.srcDir = srcDir;
this.dst = dst; this.dstDir = dstDir;
this.blockFile = blockFile;
}
public File src() {
return new File(srcDir, blockFile);
}
public File dst() {
return new File(dstDir, blockFile);
}
public String blockFile() {
return blockFile;
} }
} }
@ -1095,8 +1109,9 @@ private static void linkBlocks(File from, File to, int oldLV,
} }
final ArrayList<LinkArgs> idBasedLayoutSingleLinks = Lists.newArrayList(); final ArrayList<LinkArgs> idBasedLayoutSingleLinks = Lists.newArrayList();
linkBlocksHelper(from, to, oldLV, hl, upgradeToIdBasedLayout, to, final Map<File, File> pathCache = new HashMap<>();
idBasedLayoutSingleLinks); linkBlocksHelper(from, to, hl, upgradeToIdBasedLayout, to,
idBasedLayoutSingleLinks, pathCache);
// Detect and remove duplicate entries. // Detect and remove duplicate entries.
final ArrayList<LinkArgs> duplicates = final ArrayList<LinkArgs> duplicates =
@ -1122,7 +1137,7 @@ public Void call() throws IOException {
idBasedLayoutSingleLinks.size()); idBasedLayoutSingleLinks.size());
for (int j = iCopy; j < upperBound; j++) { for (int j = iCopy; j < upperBound; j++) {
LinkArgs cur = idBasedLayoutSingleLinks.get(j); LinkArgs cur = idBasedLayoutSingleLinks.get(j);
HardLink.createHardLink(cur.src, cur.dst); HardLink.createHardLink(cur.src(), cur.dst());
} }
return null; return null;
} }
@ -1155,9 +1170,9 @@ static ArrayList<LinkArgs> findDuplicateEntries(ArrayList<LinkArgs> all) {
@Override @Override
public int compare(LinkArgs a, LinkArgs b) { public int compare(LinkArgs a, LinkArgs b) {
return ComparisonChain.start(). return ComparisonChain.start().
compare(a.src.getName(), b.src.getName()). compare(a.blockFile(), b.blockFile()).
compare(a.src, b.src). compare(a.src(), b.src()).
compare(a.dst, b.dst). compare(a.dst(), b.dst()).
result(); result();
} }
}); });
@ -1167,8 +1182,8 @@ public int compare(LinkArgs a, LinkArgs b) {
boolean addedPrev = false; boolean addedPrev = false;
for (int i = 0; i < all.size(); i++) { for (int i = 0; i < all.size(); i++) {
LinkArgs args = all.get(i); LinkArgs args = all.get(i);
long blockId = Block.getBlockId(args.src.getName()); long blockId = Block.getBlockId(args.blockFile());
boolean isMeta = Block.isMetaFilename(args.src.getName()); boolean isMeta = Block.isMetaFilename(args.blockFile());
if ((prevBlockId == null) || if ((prevBlockId == null) ||
(prevBlockId.longValue() != blockId)) { (prevBlockId.longValue() != blockId)) {
prevBlockId = blockId; prevBlockId = blockId;
@ -1207,10 +1222,10 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
TreeMap<Long, List<LinkArgs>> highestGenstamps = TreeMap<Long, List<LinkArgs>> highestGenstamps =
new TreeMap<Long, List<LinkArgs>>(); new TreeMap<Long, List<LinkArgs>>();
for (LinkArgs duplicate : duplicates) { for (LinkArgs duplicate : duplicates) {
if (!Block.isMetaFilename(duplicate.src.getName())) { if (!Block.isMetaFilename(duplicate.blockFile())) {
continue; continue;
} }
long blockId = Block.getBlockId(duplicate.src.getName()); long blockId = Block.getBlockId(duplicate.blockFile());
List<LinkArgs> prevHighest = highestGenstamps.get(blockId); List<LinkArgs> prevHighest = highestGenstamps.get(blockId);
if (prevHighest == null) { if (prevHighest == null) {
List<LinkArgs> highest = new LinkedList<LinkArgs>(); List<LinkArgs> highest = new LinkedList<LinkArgs>();
@ -1219,8 +1234,8 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
continue; continue;
} }
long prevGenstamp = long prevGenstamp =
Block.getGenerationStamp(prevHighest.get(0).src.getName()); Block.getGenerationStamp(prevHighest.get(0).blockFile());
long genstamp = Block.getGenerationStamp(duplicate.src.getName()); long genstamp = Block.getGenerationStamp(duplicate.blockFile());
if (genstamp < prevGenstamp) { if (genstamp < prevGenstamp) {
continue; continue;
} }
@ -1234,19 +1249,19 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
// from the duplicates list. // from the duplicates list.
for (Iterator<LinkArgs> iter = duplicates.iterator(); iter.hasNext(); ) { for (Iterator<LinkArgs> iter = duplicates.iterator(); iter.hasNext(); ) {
LinkArgs duplicate = iter.next(); LinkArgs duplicate = iter.next();
long blockId = Block.getBlockId(duplicate.src.getName()); long blockId = Block.getBlockId(duplicate.blockFile());
List<LinkArgs> highest = highestGenstamps.get(blockId); List<LinkArgs> highest = highestGenstamps.get(blockId);
if (highest != null) { if (highest != null) {
boolean found = false; boolean found = false;
for (LinkArgs high : highest) { for (LinkArgs high : highest) {
if (high.src.getParent().equals(duplicate.src.getParent())) { if (high.src().getParent().equals(duplicate.src().getParent())) {
found = true; found = true;
break; break;
} }
} }
if (!found) { if (!found) {
LOG.warn("Unexpectedly low genstamp on {}.", LOG.warn("Unexpectedly low genstamp on {}.",
duplicate.src.getAbsolutePath()); duplicate.src().getAbsolutePath());
iter.remove(); iter.remove();
} }
} }
@ -1257,25 +1272,25 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
// preserving one block file / metadata file pair. // preserving one block file / metadata file pair.
TreeMap<Long, LinkArgs> longestBlockFiles = new TreeMap<Long, LinkArgs>(); TreeMap<Long, LinkArgs> longestBlockFiles = new TreeMap<Long, LinkArgs>();
for (LinkArgs duplicate : duplicates) { for (LinkArgs duplicate : duplicates) {
if (Block.isMetaFilename(duplicate.src.getName())) { if (Block.isMetaFilename(duplicate.blockFile())) {
continue; continue;
} }
long blockId = Block.getBlockId(duplicate.src.getName()); long blockId = Block.getBlockId(duplicate.blockFile());
LinkArgs prevLongest = longestBlockFiles.get(blockId); LinkArgs prevLongest = longestBlockFiles.get(blockId);
if (prevLongest == null) { if (prevLongest == null) {
longestBlockFiles.put(blockId, duplicate); longestBlockFiles.put(blockId, duplicate);
continue; continue;
} }
long blockLength = duplicate.src.length(); long blockLength = duplicate.src().length();
long prevBlockLength = prevLongest.src.length(); long prevBlockLength = prevLongest.src().length();
if (blockLength < prevBlockLength) { if (blockLength < prevBlockLength) {
LOG.warn("Unexpectedly short length on {}.", LOG.warn("Unexpectedly short length on {}.",
duplicate.src.getAbsolutePath()); duplicate.src().getAbsolutePath());
continue; continue;
} }
if (blockLength > prevBlockLength) { if (blockLength > prevBlockLength) {
LOG.warn("Unexpectedly short length on {}.", LOG.warn("Unexpectedly short length on {}.",
prevLongest.src.getAbsolutePath()); prevLongest.src().getAbsolutePath());
} }
longestBlockFiles.put(blockId, duplicate); longestBlockFiles.put(blockId, duplicate);
} }
@ -1284,21 +1299,22 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
// arbitrarily selected by us. // arbitrarily selected by us.
for (Iterator<LinkArgs> iter = all.iterator(); iter.hasNext(); ) { for (Iterator<LinkArgs> iter = all.iterator(); iter.hasNext(); ) {
LinkArgs args = iter.next(); LinkArgs args = iter.next();
long blockId = Block.getBlockId(args.src.getName()); long blockId = Block.getBlockId(args.blockFile());
LinkArgs bestDuplicate = longestBlockFiles.get(blockId); LinkArgs bestDuplicate = longestBlockFiles.get(blockId);
if (bestDuplicate == null) { if (bestDuplicate == null) {
continue; // file has no duplicates continue; // file has no duplicates
} }
if (!bestDuplicate.src.getParent().equals(args.src.getParent())) { if (!bestDuplicate.src().getParent().equals(args.src().getParent())) {
LOG.warn("Discarding {}.", args.src.getAbsolutePath()); LOG.warn("Discarding {}.", args.src().getAbsolutePath());
iter.remove(); iter.remove();
} }
} }
} }
static void linkBlocksHelper(File from, File to, int oldLV, HardLink hl, static void linkBlocksHelper(File from, File to, HardLink hl,
boolean upgradeToIdBasedLayout, File blockRoot, boolean upgradeToIdBasedLayout, File blockRoot,
List<LinkArgs> idBasedLayoutSingleLinks) throws IOException { List<LinkArgs> idBasedLayoutSingleLinks, Map<File, File> pathCache)
throws IOException {
if (!from.exists()) { if (!from.exists()) {
return; return;
} }
@ -1338,8 +1354,18 @@ public boolean accept(File dir, String name) {
throw new IOException("Failed to mkdirs " + blockLocation); throw new IOException("Failed to mkdirs " + blockLocation);
} }
} }
idBasedLayoutSingleLinks.add(new LinkArgs(new File(from, blockName), /**
new File(blockLocation, blockName))); * The destination path is 32x32, so 1024 distinct paths. Therefore
* we cache the destination path and reuse the same File object on
* potentially thousands of blocks located on this volume.
* This method is called recursively so the cache is passed through
* each recursive call. There is one cache per volume, and it is only
* accessed by a single thread so no locking is needed.
*/
File cachedDest = pathCache
.computeIfAbsent(blockLocation, k -> blockLocation);
idBasedLayoutSingleLinks.add(new LinkArgs(from,
cachedDest, blockName));
hl.linkStats.countSingleLinks++; hl.linkStats.countSingleLinks++;
} }
} else { } else {
@ -1362,8 +1388,8 @@ public boolean accept(File dir, String name) {
if (otherNames != null) { if (otherNames != null) {
for (int i = 0; i < otherNames.length; i++) { for (int i = 0; i < otherNames.length; i++) {
linkBlocksHelper(new File(from, otherNames[i]), linkBlocksHelper(new File(from, otherNames[i]),
new File(to, otherNames[i]), oldLV, hl, upgradeToIdBasedLayout, new File(to, otherNames[i]), hl, upgradeToIdBasedLayout,
blockRoot, idBasedLayoutSingleLinks); blockRoot, idBasedLayoutSingleLinks, pathCache);
} }
} }
} }