big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-7795. Show warning if not all favored nodes were chosen by namenode. Contributed by Kihwal Lee.

Browse Source
This commit is contained in:
Kihwal Lee 2015-02-17 13:05:43 -06:00
parent 78a7e8d3a6
commit db6606223c
2 changed files with 21 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -644,6 +644,9 @@ Release 2.7.0 - UNRELEASED
HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh) HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh)
HDFS-7795. Show warning if not all favored nodes were chosen by namenode
(kihwal)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode. HDFS-7454. Reduce memory footprint for AclEntries in NameNode.

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
View File

@ -35,6 +35,7 @@
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.HashSet;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -1443,7 +1444,7 @@ private boolean createBlockOutputStream(DatanodeInfo[] nodes,
ExtendedBlock blockCopy = new ExtendedBlock(block); ExtendedBlock blockCopy = new ExtendedBlock(block);
blockCopy.setNumBytes(blockSize); blockCopy.setNumBytes(blockSize);
boolean[] targetPinnings = getPinnings(nodes); boolean[] targetPinnings = getPinnings(nodes, true);
// send the request // send the request
new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken, new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs, dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
@ -1537,20 +1538,29 @@ private boolean createBlockOutputStream(DatanodeInfo[] nodes,
} }
} }
private boolean[] getPinnings(DatanodeInfo[] nodes) { private boolean[] getPinnings(DatanodeInfo[] nodes, boolean shouldLog) {
if (favoredNodes == null) { if (favoredNodes == null) {
return null; return null;
} else { } else {
boolean[] pinnings = new boolean[nodes.length]; boolean[] pinnings = new boolean[nodes.length];
HashSet<String> favoredSet =
new HashSet<String>(Arrays.asList(favoredNodes));
for (int i = 0; i < nodes.length; i++) { for (int i = 0; i < nodes.length; i++) {
pinnings[i] = false; pinnings[i] = favoredSet.remove(nodes[i].getXferAddrWithHostname());
for (int j = 0; j < favoredNodes.length; j++) { if (DFSClient.LOG.isDebugEnabled()) {
if (nodes[i].getXferAddrWithHostname().equals(favoredNodes[j])) { DFSClient.LOG.debug(nodes[i].getXferAddrWithHostname() +
pinnings[i] = true; " was chosen by name node (favored=" + pinnings[i] +
break; ").");
}
} }
} }
if (shouldLog && !favoredSet.isEmpty()) {
// There is one or more favored nodes that were not allocated.
DFSClient.LOG.warn(
"These favored nodes were specified but not chosen: " +
favoredSet +
" Specified favored nodes: " + Arrays.toString(favoredNodes));
}
return pinnings; return pinnings;
} }
} }