big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-7772

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1189847 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Steve Loughran 2011-10-27 16:29:44 +00:00
parent 1b8511ef89
commit a3c0a0e799
7 changed files with 242 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -1008,6 +1008,8 @@ Release 0.22.0 - Unreleased
HADOOP-7325. The hadoop command should not accept class names starting with
a hyphen. (Brock Noland via todd)
HADOOP-7772. javadoc the topology classes (stevel)
OPTIMIZATIONS
HADOOP-6884. Add LOG.isDebugEnabled() guard for each LOG.debug(..).

29
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
View File

@ -38,13 +38,17 @@ public class CachedDNSToSwitchMapping implements DNSToSwitchMapping {
private Map<String, String> cache = new ConcurrentHashMap<String, String>();
protected DNSToSwitchMapping rawMapping;
/**
* cache a raw DNS mapping
* @param rawMapping the raw mapping to cache
*/
public CachedDNSToSwitchMapping(DNSToSwitchMapping rawMapping) {
this.rawMapping = rawMapping;
}
/**
* Returns the hosts from 'names' that have not been cached previously
* @param names a list of hostnames to probe for being cached
* @return the hosts from 'names' that have not been cached previously
*/
private List<String> getUncachedHosts(List<String> names) {
// find out all names without cached resolved location
@ -58,7 +62,12 @@ private List<String> getUncachedHosts(List<String> names) {
}
/**
* Caches the resolved hosts
* Caches the resolved host:rack mappings. The two list
* parameters must be of equal size.
*
* @param uncachedHosts a list of hosts that were uncached
* @param resolvedHosts a list of resolved host entries where the element
* at index(i) is the resolved value for the entry in uncachedHosts[i]
*/
private void cacheResolvedHosts(List<String> uncachedHosts,
List<String> resolvedHosts) {
@ -71,8 +80,9 @@ private void cacheResolvedHosts(List<String> uncachedHosts,
}
/**
* Returns the cached resolution of the list of hostnames/addresses.
* Returns null if any of the names are not currently in the cache
* @param names a list of hostnames to look up (can be be empty)
* @return the cached resolution of the list of hostnames/addresses.
* or null if any of the names are not currently in the cache
*/
private List<String> getCachedHosts(List<String> names) {
List<String> result = new ArrayList<String>(names.size());
@ -88,6 +98,7 @@ private List<String> getCachedHosts(List<String> names) {
return result;
}
@Override
public List<String> resolve(List<String> names) {
// normalize all input names to be in the form of IP addresses
names = NetUtils.normalizeHostNames(names);
@ -97,12 +108,14 @@ public List<String> resolve(List<String> names) {
return result;
}
List<String> uncachedHosts = this.getUncachedHosts(names);
List<String> uncachedHosts = getUncachedHosts(names);
// Resolve the uncached hosts
List<String> resolvedHosts = rawMapping.resolve(uncachedHosts);
this.cacheResolvedHosts(uncachedHosts, resolvedHosts);
return this.getCachedHosts(names);
//cache them
cacheResolvedHosts(uncachedHosts, resolvedHosts);
//now look up the entire list in the cache
return getCachedHosts(names);
}
}

7
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
View File

@ -23,7 +23,7 @@
import org.apache.hadoop.classification.InterfaceStability;
/**
* An interface that should be implemented to allow pluggable
* An interface that must be implemented to allow pluggable
* DNS-name/IP-address to RackID resolvers.
*
*/
@ -40,8 +40,9 @@ public interface DNSToSwitchMapping {
* Note the hostname/ip-address is not part of the returned path.
* The network topology of the cluster would determine the number of
* components in the network path.
* @param names
* @return list of resolved network paths
* @param names the list of hosts to resolve (can be empty)
* @return list of resolved network paths.
* If <i>names</i> is empty, the returned list is also empty
*/
public List<String> resolve(List<String> names);
}

64
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
View File

@ -45,8 +45,8 @@ public class NetworkTopology {
public static final Log LOG =
LogFactory.getLog(NetworkTopology.class);
/* Inner Node represent a switch/router of a data center or rack.
* Different from a leave node, it has non-null children.
/** InnerNode represents a switch/router of a data center or rack.
* Different from a leaf node, it has non-null children.
*/
private class InnerNode extends NodeBase {
private ArrayList<Node> children=new ArrayList<Node>();
@ -68,16 +68,16 @@ private class InnerNode extends NodeBase {
super(name, location, parent, level);
}
/** Get its children */
/** @return its children */
Collection<Node> getChildren() {return children;}
/** Return the number of children this node has */
/** @return the number of children this node has */
int getNumOfChildren() {
return children.size();
}
/** Judge if this node represents a rack
* Return true if it has no child or its children are not InnerNodes
* @return true if it has no child or its children are not InnerNodes
*/
boolean isRack() {
if (children.isEmpty()) {
@ -225,7 +225,11 @@ boolean remove(Node n) {
}
} // end of remove
/** Given a node's string representation, return a reference to the node */
/** Given a node's string representation, return a reference to the node
* @param loc string location of the form /rack/node
* @return null if the node is not found or the childnode is there but
* not an instance of {@link InnerNode}
*/
private Node getLoc(String loc) {
if (loc == null || loc.length() == 0) return this;
@ -246,7 +250,12 @@ private Node getLoc(String loc) {
}
/** get <i>leafIndex</i> leaf of this subtree
* if it is not in the <i>excludedNode</i>*/
* if it is not in the <i>excludedNode</i>
*
* @param leafIndex an indexed leaf of the node
* @param excludedNode an excluded node (can be null)
* @return
*/
private Node getLeaf(int leafIndex, Node excludedNode) {
int count=0;
// check if the excluded node a leaf
@ -298,8 +307,13 @@ int getNumOfLeaves() {
}
} // end of InnerNode
InnerNode clusterMap = new InnerNode(InnerNode.ROOT); // the root
private int numOfRacks = 0; // rack counter
/**
* the root cluster map
*/
InnerNode clusterMap = new InnerNode(InnerNode.ROOT);
/** rack counter */
private int numOfRacks = 0;
/** the lock used to manage access */
private ReadWriteLock netlock;
public NetworkTopology() {
@ -308,8 +322,7 @@ public NetworkTopology() {
/** Add a leaf node
* Update node counter & rack counter if necessary
* @param node
* node to be added
* @param node node to be added; can be null
* @exception IllegalArgumentException if add a node to a leave
or node to be added is not a leaf
*/
@ -342,9 +355,8 @@ public void add(Node node) {
}
/** Remove a node
* Update node counter & rack counter if necessary
* @param node
* node to be removed
* Update node counter and rack counter if necessary
* @param node node to be removed; can be null
*/
public void remove(Node node) {
if (node==null) return;
@ -371,8 +383,7 @@ public void remove(Node node) {
/** Check if the tree contains node <i>node</i>
*
* @param node
* a node
* @param node a node
* @return true if <i>node</i> is already in the tree; false otherwise
*/
public boolean contains(Node node) {
@ -380,11 +391,12 @@ public boolean contains(Node node) {
netlock.readLock().lock();
try {
Node parent = node.getParent();
for(int level=node.getLevel(); parent!=null&&level>0;
parent=parent.getParent(), level--) {
if (parent == clusterMap)
for (int level = node.getLevel(); parent != null && level > 0;
parent = parent.getParent(), level--) {
if (parent == clusterMap) {
return true;
}
}
} finally {
netlock.readLock().unlock();
}
@ -409,7 +421,7 @@ public Node getNode(String loc) {
}
}
/** Return the total number of racks */
/** @return the total number of racks */
public int getNumOfRacks() {
netlock.readLock().lock();
try {
@ -419,7 +431,7 @@ public int getNumOfRacks() {
}
}
/** Return the total number of nodes */
/** @return the total number of leaf nodes */
public int getNumOfLeaves() {
netlock.readLock().lock();
try {
@ -435,8 +447,8 @@ public int getNumOfLeaves() {
* to their closest common ancestor.
* @param node1 one node
* @param node2 another node
* @return the distance between node1 and node2
* node1 or node2 do not belong to the cluster
* @return the distance between node1 and node2 which is zero if they are the same
* or {@link Integer#MAX_VALUE} if node1 or node2 do not belong to the cluster
*/
public int getDistance(Node node1, Node node2) {
if (node1 == node2) {
@ -477,8 +489,8 @@ public int getDistance(Node node1, Node node2) {
}
/** Check if two nodes are on the same rack
* @param node1 one node
* @param node2 another node
* @param node1 one node (can be null)
* @param node2 another node (can be null)
* @return true if node1 and node2 are on the same rack; false otherwise
* @exception IllegalArgumentException when either node1 or node2 is null, or
* node1 or node2 do not belong to the cluster
@ -622,6 +634,8 @@ static private void swap(Node[] nodes, int i, int j) {
* If neither local node or local rack node is found, put a random replica
* location at position 0.
* It leaves the rest nodes untouched.
* @param reader the node that wishes to read a block from one of the nodes
* @param nodes the list of nodes containing data for the reader
*/
public void pseudoSortByDistance( Node reader, Node[] nodes ) {
int tempIndex = 0;

25
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/Node.java
View File

@ -33,20 +33,31 @@
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Unstable
public interface Node {
/** Return the string representation of this node's network location */
/** @return the string representation of this node's network location */
public String getNetworkLocation();
/** Set the node's network location */
/** Set this node's network location
* @param location the location
*/
public void setNetworkLocation(String location);
/** Return this node's name */
/** @return this node's name */
public String getName();
/** Return this node's parent */
/** @return this node's parent */
public Node getParent();
/** Set this node's parent */
/** Set this node's parent
* @param parent the parent
*/
public void setParent(Node parent);
/** Return this node's level in the tree.
/** @return this node's level in the tree.
* E.g. the root of a tree returns 0 and its children return 1
*/
public int getLevel();
/** Set this node's level in the tree.*/
/** Set this node's level in the tree
* @param i the level
*/
public void setLevel(int i);
}

61
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
View File

@ -27,9 +27,12 @@
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Unstable
public class NodeBase implements Node {
/** Path separator {@value} */
public final static char PATH_SEPARATOR = '/';
/** Path separator as a string {@value} */
public final static String PATH_SEPARATOR_STR = "/";
public final static String ROOT = ""; // string representation of root
/** string representation of root {@value} */
public final static String ROOT = "";
protected String name; //host:port#
protected String location; //string representation of this node's location
@ -55,7 +58,7 @@ public NodeBase(String path) {
}
/** Construct a node from its name and its location
* @param name this node's name
* @param name this node's name (can be null, must not contain {@link #PATH_SEPARATOR})
* @param location this node's location
*/
public NodeBase(String name, String location) {
@ -63,7 +66,7 @@ public NodeBase(String name, String location) {
}
/** Construct a node from its name and its location
* @param name this node's name
* @param name this node's name (can be null, must not contain {@link #PATH_SEPARATOR})
* @param location this node's location
* @param parent this node's parent node
* @param level this node's level in the tree
@ -74,7 +77,11 @@ public NodeBase(String name, String location, Node parent, int level) {
this.level = level;
}
/* set this node's name and location */
/**
* set this node's name and location
* @param name the (nullable) name -which cannot contain the {@link #PATH_SEPARATOR}
* @param location the location
*/
private void set(String name, String location) {
if (name != null && name.contains(PATH_SEPARATOR_STR))
throw new IllegalArgumentException(
@ -83,27 +90,43 @@ private void set(String name, String location) {
this.location = location;
}
/** Return this node's name */
/** @return this node's name */
@Override
public String getName() { return name; }
/** Return this node's network location */
/** @return this node's network location */
@Override
public String getNetworkLocation() { return location; }
/** Set this node's network location */
/** Set this node's network location
* @param location the location
*/
@Override
public void setNetworkLocation(String location) { this.location = location; }
/** Return this node's path */
/**
* Get the path of a node
* @param node a non-null node
* @return the path of a node
*/
public static String getPath(Node node) {
return node.getNetworkLocation()+PATH_SEPARATOR_STR+node.getName();
}
/** Return this node's string representation */
/** @return this node's path as its string representation */
@Override
public String toString() {
return getPath(this);
}
/** Normalize a path */
static public String normalize(String path) {
/** Normalize a path by stripping off any trailing {@link #PATH_SEPARATOR}
* @param path path to normalize.
* @return the normalised path
* If <i>path</i>is null or empty {@link #ROOT} is returned
* @throws IllegalArgumentException if the first character of a non empty path
* is not {@link #PATH_SEPARATOR}
*/
public static String normalize(String path) {
if (path == null || path.length() == 0) return ROOT;
if (path.charAt(0) != PATH_SEPARATOR) {
@ -119,20 +142,28 @@ static public String normalize(String path) {
return path;
}
/** Return this node's parent */
/** @return this node's parent */
@Override
public Node getParent() { return parent; }
/** Set this node's parent */
/** Set this node's parent
* @param parent the parent
*/
@Override
public void setParent(Node parent) {
this.parent = parent;
}
/** Return this node's level in the tree.
/** @return this node's level in the tree.
* E.g. the root of a tree returns 0 and its children return 1
*/
@Override
public int getLevel() { return level; }
/** Set this node's level in the tree */
/** Set this node's level in the tree
* @param level the level
*/
@Override
public void setLevel(int level) {
this.level = level;
}

50
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
View File

@ -32,7 +32,7 @@
/**
* This class implements the {@link DNSToSwitchMapping} interface using a
* script configured via net.topology.script.file.name .
* script configured via the {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY}
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
@ -43,30 +43,51 @@ public ScriptBasedMapping() {
super(new RawScriptBasedMapping());
}
// script must accept at least this many args
/**
* Minimum number of arguments: {@value}
*/
static final int MIN_ALLOWABLE_ARGS = 1;
/**
* Default number of arguments: {@value}
*/
static final int DEFAULT_ARG_COUNT =
CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_DEFAULT;
/**
* key to the script filename {@value}
*/
static final String SCRIPT_FILENAME_KEY =
CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY ;
/**
* key to the argument count that the script supports
*/
static final String SCRIPT_ARG_COUNT_KEY =
CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY ;
/**
* Create an instance from the given configuration
* @param conf configuration
*/
public ScriptBasedMapping(Configuration conf) {
this();
setConf(conf);
}
@Override
public Configuration getConf() {
return ((RawScriptBasedMapping)rawMapping).getConf();
}
@Override
public void setConf(Configuration conf) {
((RawScriptBasedMapping)rawMapping).setConf(conf);
}
/**
* This is the uncached script mapping that is fed into the cache managed
* by the superclass {@link CachedDNSToSwitchMapping}
*/
private static final class RawScriptBasedMapping
implements DNSToSwitchMapping {
private String scriptName;
@ -74,17 +95,32 @@ private static final class RawScriptBasedMapping
private int maxArgs; //max hostnames per call of the script
private static Log LOG =
LogFactory.getLog(ScriptBasedMapping.class);
/**
* Set the configuration and
* @param conf extract the configuration parameters of interest
*/
public void setConf (Configuration conf) {
this.scriptName = conf.get(SCRIPT_FILENAME_KEY);
this.maxArgs = conf.getInt(SCRIPT_ARG_COUNT_KEY, DEFAULT_ARG_COUNT);
this.conf = conf;
}
/**
* Get the configuration
* @return the configuration
*/
public Configuration getConf () {
return conf;
}
/**
* Constructor. The mapping is not ready to use until
* {@link #setConf(Configuration)} has been called
*/
public RawScriptBasedMapping() {}
@Override
public List<String> resolve(List<String> names) {
List <String> m = new ArrayList<String>(names.size());
@ -123,6 +159,14 @@ public List<String> resolve(List<String> names) {
return m;
}
/**
* Build and execute the resolution command. The command is
* executed in the directory specified by the system property
* "user.dir" if set; otherwise the current working directory is used
* @param args a list of arguments
* @return null if the number of arguments is out of range,
* or the output of the command.
*/
private String runResolveCommand(List<String> args) {
int loopCount = 0;
if (args.size() == 0) {
@ -139,7 +183,7 @@ private String runResolveCommand(List<String> args) {
while (numProcessed != args.size()) {
int start = maxArgs * loopCount;
List <String> cmdList = new ArrayList<String>();
List<String> cmdList = new ArrayList<String>();
cmdList.add(scriptName);
for (numProcessed = start; numProcessed < (start + maxArgs) &&
numProcessed < args.size(); numProcessed++) {