big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11370. Optimize NamenodeFsck#getReplicaInfo. Contributed Takanobu Asanuma.

Browse Source
This commit is contained in:
Jing Zhao 2017-02-01 11:21:35 -08:00
parent 59c5f18784
commit b6f290d5b6
2 changed files with 41 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
View File

@ -24,7 +24,9 @@
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.COMPLETE;
@ -109,6 +111,29 @@ public DatanodeStorageInfo[] getExpectedStorageLocations() {
return storages;
}
/**
* Note that this iterator doesn't guarantee thread-safe. It depends on
* external mechanisms such as the FSNamesystem lock for protection.
*/
public Iterator<DatanodeStorageInfo> getExpectedStorageLocationsIterator() {
return new Iterator<DatanodeStorageInfo>() {
private int index = 0;
@Override
public boolean hasNext() {
return index < replicas.length;
}
@Override
public DatanodeStorageInfo next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
return replicas[index++].getExpectedStorageLocation();
}
};
}
/**
* @return the index array indicating the block index in each storage. Used
* only by striped blocks.

28
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
View File

@ -589,23 +589,27 @@ private String getReplicaInfo(BlockInfo storedBlock) {
return "";
}
final boolean isComplete = storedBlock.isComplete();
DatanodeStorageInfo[] storages = isComplete ?
blockManager.getStorages(storedBlock) :
storedBlock.getUnderConstructionFeature().getExpectedStorageLocations();
Iterator<DatanodeStorageInfo> storagesItr;
StringBuilder sb = new StringBuilder(" [");
final boolean isStriped = storedBlock.isStriped();
Map<DatanodeStorageInfo, Long> storage2Id = new HashMap<>();
if (isStriped && isComplete) {
long blockId = storedBlock.getBlockId();
Iterable<StorageAndBlockIndex> sis =
((BlockInfoStriped)storedBlock).getStorageAndIndexInfos();
for (StorageAndBlockIndex si: sis){
storage2Id.put(si.getStorage(), blockId + si.getBlockIndex());
if (isComplete) {
if (isStriped) {
long blockId = storedBlock.getBlockId();
Iterable<StorageAndBlockIndex> sis =
((BlockInfoStriped) storedBlock).getStorageAndIndexInfos();
for (StorageAndBlockIndex si : sis) {
storage2Id.put(si.getStorage(), blockId + si.getBlockIndex());
}
}
storagesItr = storedBlock.getStorageInfos();
} else {
storagesItr = storedBlock.getUnderConstructionFeature()
.getExpectedStorageLocationsIterator();
}
for (int i = 0; i < storages.length; i++) {
DatanodeStorageInfo storage = storages[i];
while (storagesItr.hasNext()) {
DatanodeStorageInfo storage = storagesItr.next();
if (isStriped && isComplete) {
long index = storage2Id.get(storage);
sb.append("blk_" + index + ":");
@ -649,7 +653,7 @@ private String getReplicaInfo(BlockInfo storedBlock) {
sb.append("LIVE)");
}
}
if (i < storages.length - 1) {
if (storagesItr.hasNext()) {
sb.append(", ");
}
}