Configured
+ * base class, and should not be changed to do so, as it causes problems
+ * for subclasses. The constructor of the Configured calls
+ * the {@link #setConf(Configuration)} method, which will call into the
+ * subclasses before they have been fully constructed.
+ *
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class AbstractDNSToSwitchMapping
+ implements DNSToSwitchMapping, Configurable {
+
+ private Configuration conf;
+
+ /**
+ * Create an unconfigured instance
+ */
+ protected AbstractDNSToSwitchMapping() {
+ }
+
+ /**
+ * Create an instance, caching the configuration file.
+ * This constructor does not call {@link #setConf(Configuration)}; if
+ * a subclass extracts information in that method, it must call it explicitly.
+ * @param conf the configuration
+ */
+ protected AbstractDNSToSwitchMapping(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Predicate that indicates that the switch mapping is known to be
+ * single-switch. The base class returns false: it assumes all mappings are
+ * multi-rack. Subclasses may override this with methods that are more aware
+ * of their topologies.
+ *
+ *
+ *
+ * This method is used when parts of Hadoop need know whether to apply
+ * single rack vs multi-rack policies, such as during block placement.
+ * Such algorithms behave differently if they are on multi-switch systems.
+ *
+ *
+ * @return true if the mapping thinks that it is on a single switch
+ */
+ public boolean isSingleSwitch() {
+ return false;
+ }
+
+ /**
+ * Query for a {@link DNSToSwitchMapping} instance being on a single
+ * switch.
+ *
+ * This predicate simply assumes that all mappings not derived from
+ * this class are multi-switch.
+ * @param mapping the mapping to query
+ * @return true if the base class says it is single switch, or the mapping
+ * is not derived from this class.
+ */
+ public static boolean isMappingSingleSwitch(DNSToSwitchMapping mapping) {
+ return mapping instanceof AbstractDNSToSwitchMapping
+ && ((AbstractDNSToSwitchMapping) mapping).isSingleSwitch();
+ }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
index f29e53cfbf..bdfc95cb0e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
@@ -34,9 +34,13 @@
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public class CachedDNSToSwitchMapping implements DNSToSwitchMapping {
+public class CachedDNSToSwitchMapping extends AbstractDNSToSwitchMapping {
private MapRawScriptBasedMapping that performs
+ * the work: reading the configuration parameters, executing any defined
+ * script, handling errors and such like. The outer
+ * class extends {@link CachedDNSToSwitchMapping} to cache the delegated
+ * queries.
+ *
+ * This DNS mapper's {@link #isSingleSwitch()} predicate returns
+ * true if and only if a script is defined.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public final class ScriptBasedMapping extends CachedDNSToSwitchMapping
-implements Configurable
-{
- public ScriptBasedMapping() {
- super(new RawScriptBasedMapping());
- }
+public final class ScriptBasedMapping extends CachedDNSToSwitchMapping {
/**
* Minimum number of arguments: {@value}
@@ -65,6 +70,18 @@ public ScriptBasedMapping() {
static final String SCRIPT_ARG_COUNT_KEY =
CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY ;
+ /**
+ * Create an instance with the default configuration.
+ *
+ * Calling {@link #setConf(Configuration)} will trigger a
+ * re-evaluation of the configuration settings and so be used to
+ * set up the mapping script.
+ *
+ */
+ public ScriptBasedMapping() {
+ super(new RawScriptBasedMapping());
+ }
+
/**
* Create an instance from the given configuration
* @param conf configuration
@@ -74,14 +91,31 @@ public ScriptBasedMapping(Configuration conf) {
setConf(conf);
}
- @Override
- public Configuration getConf() {
- return ((RawScriptBasedMapping)rawMapping).getConf();
+ /**
+ * Get the cached mapping and convert it to its real type
+ * @return the inner raw script mapping.
+ */
+ private RawScriptBasedMapping getRawMapping() {
+ return (RawScriptBasedMapping)rawMapping;
}
+ @Override
+ public Configuration getConf() {
+ return getRawMapping().getConf();
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * This will get called in the superclass constructor, so a check is needed
+ * to ensure that the raw mapping is defined before trying to relaying a null
+ * configuration.
+ * @param conf
+ */
@Override
public void setConf(Configuration conf) {
- ((RawScriptBasedMapping)rawMapping).setConf(conf);
+ super.setConf(conf);
+ getRawMapping().setConf(conf);
}
/**
@@ -89,29 +123,26 @@ public void setConf(Configuration conf) {
* by the superclass {@link CachedDNSToSwitchMapping}
*/
private static final class RawScriptBasedMapping
- implements DNSToSwitchMapping {
+ extends AbstractDNSToSwitchMapping {
private String scriptName;
- private Configuration conf;
private int maxArgs; //max hostnames per call of the script
- private static Log LOG =
+ private static final Log LOG =
LogFactory.getLog(ScriptBasedMapping.class);
/**
- * Set the configuration and
- * @param conf extract the configuration parameters of interest
+ * Set the configuration and extract the configuration parameters of interest
+ * @param conf the new configuration
*/
+ @Override
public void setConf (Configuration conf) {
- this.scriptName = conf.get(SCRIPT_FILENAME_KEY);
- this.maxArgs = conf.getInt(SCRIPT_ARG_COUNT_KEY, DEFAULT_ARG_COUNT);
- this.conf = conf;
- }
-
- /**
- * Get the configuration
- * @return the configuration
- */
- public Configuration getConf () {
- return conf;
+ super.setConf(conf);
+ if (conf != null) {
+ scriptName = conf.get(SCRIPT_FILENAME_KEY);
+ maxArgs = conf.getInt(SCRIPT_ARG_COUNT_KEY, DEFAULT_ARG_COUNT);
+ } else {
+ scriptName = null;
+ maxArgs = 0;
+ }
}
/**
@@ -122,42 +153,42 @@ public RawScriptBasedMapping() {}
@Override
public Listhost1=r1,host2=r1,host3=r2.
+ *
+ * Important: spaces not trimmed and are considered significant.
+ */
+ public static final String KEY_HADOOP_CONFIGURED_NODE_MAPPING =
+ "hadoop.configured.node.mapping";
+
+ /**
+ * Configure the mapping by extracting any mappings defined in the
+ * {@link #KEY_HADOOP_CONFIGURED_NODE_MAPPING} field
+ * @param conf new configuration
+ */
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ if (conf != null) {
+ String[] mappings = conf.getStrings(KEY_HADOOP_CONFIGURED_NODE_MAPPING);
+ if (mappings != null) {
+ for (String str : mappings) {
+ String host = str.substring(0, str.indexOf('='));
+ String rack = str.substring(str.indexOf('=') + 1);
+ addNodeToRack(host, rack);
+ }
}
}
}
- /* Only one instance per JVM */
- private static MapMonospaced | Used for commands, HTTP request and responses and code blocks. |
<Monospaced> | User entered values. |
[Monospaced] | Optional values. When the value is not specified, the default value is used. |
| Italics | Important phrases and words. |
+ The HTTP REST API supports the complete FileSystem interface for HDFS. + The operations and the corresponding FileSystem methods are shown in the next section. + The Section HTTP Query Parameter Dictionary specifies the parameter details + such as the defaults and the valid values. +
+OPEN
+ (see FileSystem.open)
+ GETFILESTATUS
+ (see FileSystem.getFileStatus)
+ LISTSTATUS
+ (see FileSystem.listStatus)
+ GETCONTENTSUMMARY
+ (see FileSystem.getContentSummary)
+ GETFILECHECKSUM
+ (see FileSystem.getFileChecksum)
+ GETHOMEDIRECTORY
+ (see FileSystem.getHomeDirectory)
+ GETDELEGATIONTOKEN
+ (see FileSystem.getDelegationToken)
+ CREATE
+ (see FileSystem.create)
+ MKDIRS
+ (see FileSystem.mkdirs)
+ RENAME
+ (see FileSystem.rename)
+ SETREPLICATION
+ (see FileSystem.setReplication)
+ SETOWNER
+ (see FileSystem.setOwner)
+ SETPERMISSION
+ (see FileSystem.setPermission)
+ SETTIMES
+ (see FileSystem.setTimes)
+ RENEWDELEGATIONTOKEN
+ (see DistributedFileSystem.renewDelegationToken)
+ CANCELDELEGATIONTOKEN
+ (see DistributedFileSystem.cancelDelegationToken)
+ APPEND
+ (see FileSystem.append)
+ DELETE
+ (see FileSystem.delete)
+
+ The FileSystem scheme of WebHDFS is "webhdfs://".
+ A WebHDFS FileSystem URI has the following format.
+
+ The above WebHDFS URI corresponds to the below HDFS URI. +
+
+ In the REST API, the prefix "/webhdfs/v1" is inserted in the path and a query is appended at the end.
+ Therefore, the corresponding HTTP URL has the following format.
+
+ When security is off, the authenticated user is the username specified in the user.name query parameter.
+ If the user.name parameter is not set,
+ the server may either set the authenticated user to a default web user, if there is any, or return an error response.
+
+ When security is on, authentication is performed by either Hadoop delegation token or Kerberos SPNEGO.
+ If a token is set in the delegation query parameter, the authenticated user is the user encoded in the token.
+ If the delegation parameter is not set, the user is authenticated by Kerberos SPNEGO.
+
+Below are examples using the curl command tool.
+
+ When the proxy user feature is enabled, a proxy user P may submit a request on behalf of another user U.
+ The username of U must be specified in the doas query parameter unless a delegation token is presented in authentication.
+ In such case, the information of both users P and U must be encoded in the delegation token.
+
Location header with the file data to be written.
+ 201 Created response with zero content length
+and the WebHDFS URI of the file in the Location header:
+
+ Note that the reason of having two-step create/append is
+ for preventing clients to send out data before the redirect.
+ This issue is addressed by the "Expect: 100-continue" header in HTTP/1.1;
+ see RFC 2616, Section 8.2.3.
+ Unfortunately, there are software library bugs (e.g. Jetty 6 HTTP server and Java 6 HTTP client),
+ which do not correctly implement "Expect: 100-continue".
+ The two-step create/append is a temporary workaround for the software library bugs.
+
+ See also:
+ overwrite,
+ blocksize,
+ replication,
+ permission,
+ buffersize,
+ FileSystem.create
+
Location header with the file data to be appended.
+ + See the note in the previous section for the description of why this operation requires two steps. +
+
+ See also:
+ buffersize,
+ FileSystem.append
+
+ See also:
+ offset,
+ length,
+ buffersize,
+ FileSystem.open
+
boolean JSON object:
+
+ See also:
+ permission,
+ FileSystem.mkdirs
+
boolean JSON object:
+
+ See also:
+ destination,
+ FileSystem.rename
+
boolean JSON object:
+
+ See also:
+ recursive,
+ FileSystem.delete
+
FileStatus JSON object:
+ + See also: + FileSystem.getFileStatus +
+FileStatuses JSON object:
+ + See also: + FileSystem.listStatus +
+ContentSummary JSON object:
+ + See also: + FileSystem.getContentSummary +
+FileChecksum JSON object:
+ + See also: + FileSystem.getFileChecksum +
+Path JSON object:
+ + See also: + FileSystem.getHomeDirectory +
+
+ See also:
+ permission,
+ FileSystem.setPermission
+
+ See also:
+ owner,
+ group,
+ FileSystem.setOwner
+
boolean JSON object:
+
+ See also:
+ replication,
+ FileSystem.setReplication
+
+ See also:
+ modificationtime,
+ accesstime,
+ FileSystem.setTimes
+
Token JSON object:
+
+ See also:
+ renewer,
+ FileSystem.getDelegationToken
+
long JSON object:
+
+ See also:
+ token,
+ DistributedFileSystem.renewDelegationToken
+
+ See also:
+ token,
+ DistributedFileSystem.cancelDelegationToken
+
+ When an operation fails, the server may throw an exception.
+ The JSON schema of error responses is defined in RemoteException JSON schema.
+ The table below shows the mapping from exceptions to HTTP response codes.
+
| Exceptions | HTTP Response Codes |
|---|---|
IllegalArgumentException | 400 Bad Request |
UnsupportedOperationException | 400 Bad Request |
SecurityException | 401 Unauthorized |
IOException | 403 Forbidden |
FileNotFoundException | 404 Not Found |
RumtimeException | 500 Internal Server Error |
+ Below are examples of exception responses. +
+ +
+All operations, except for OPEN,
+either return a zero-length response or a JSON response .
+For OPEN, the response is an octet-stream.
+The JSON schemas are shown below.
+See draft-zyp-json-schema-03
+for the syntax definitions of the JSON schemas.
+
+ See also:
+ MKDIRS,
+ RENAME,
+ DELETE,
+ SETREPLICATION
+
+ See also:
+ GETCONTENTSUMMARY
+
+ See also:
+ GETFILECHECKSUM
+
+ See also:
+ GETFILESTATUS,
+ FileStatus
+
+ JavaScript syntax is used to define fileStatusProperties
+ so that it can be referred in both FileStatus and FileStatuses JSON schemas.
+
+ A FileStatuses JSON object represents an array of FileStatus JSON objects.
+
+ See also:
+ LISTSTATUS,
+ FileStatus
+
+ See also:
+ RENEWDELEGATIONTOKEN,
+
+ See also:
+ GETHOMEDIRECTORY,
+ Path
+
+ See also:
+ GETDELEGATIONTOKEN,
+ the note in Delegation.
+
| Name | accesstime |
| Description | The access time of a file/directory. |
| Type | long |
| Default Value | -1 (means keeping it unchanged) |
| Valid Values | -1 or a timestamp |
| Syntax | Any integer. |
+ See also:
+ SETTIMES
+
| Name | blocksize |
| Description | The block size of a file. |
| Type | long |
| Default Value | Specified in the configuration. |
| Valid Values | > 0 |
| Syntax | Any integer. |
+ See also:
+ CREATE
+
| Name | buffersize |
| Description | The size of the buffer used in transferring data. |
| Type | int |
| Default Value | Specified in the configuration. |
| Valid Values | > 0 |
| Syntax | Any integer. |
| Name | delegation |
| Description | The delegation token used for authentication. |
| Type | String |
| Default Value | <empty> |
| Valid Values | An encoded token. |
| Syntax | See the note below. |
+ Note that delegation tokens are encoded as a URL safe string;
+ see encodeToUrlString()
+ and decodeFromUrlString(String)
+ in org.apache.hadoop.security.token.Token for the details of the encoding.
+
+ See also: + Authentication +
+| Name | destination |
| Description | The destination path used in RENAME. |
| Type | Path |
| Default Value | <empty> (an invalid path) |
| Valid Values | An absolute FileSystem path without scheme and authority. |
| Syntax | Any path. |
+ See also:
+ RENAME
+
| Name | doas |
| Description | Allowing a proxy user to do as another user. |
| Type | String |
| Default Value | null |
| Valid Values | Any valid username. |
| Syntax | Any string. |
+ See also: + Proxy Users +
+| Name | group |
| Description | The name of a group. |
| Type | String |
| Default Value | <empty> (means keeping it unchanged) |
| Valid Values | Any valid group name. |
| Syntax | Any string. |
+ See also:
+ SETOWNER
+
| Name | length |
| Description | The number of bytes to be processed. |
| Type | long |
| Default Value | null (means the entire file) |
| Valid Values | >= 0 or null |
| Syntax | Any integer. |
+ See also:
+ OPEN
+
| Name | modificationtime |
| Description | The modification time of a file/directory. |
| Type | long |
| Default Value | -1 (means keeping it unchanged) |
| Valid Values | -1 or a timestamp |
| Syntax | Any integer. |
+ See also:
+ SETTIMES
+
| Name | offset |
| Description | The starting byte position. |
| Type | long |
| Default Value | 0 |
| Valid Values | >= 0 |
| Syntax | Any integer. |
+ See also:
+ OPEN
+
| Name | op |
| Description | The name of the operation to be executed. |
| Type | enum |
| Default Value | null (an invalid value) |
| Valid Values | Any valid operation name. |
| Syntax | Any string. |
+ See also: + Operations +
+| Name | overwrite |
| Description | If a file already exists, should it be overwritten? |
| Type | boolean |
| Default Value | false |
| Valid Values | true | false |
| Syntax | true | false |
+ See also:
+ CREATE
+
| Name | owner |
| Description | The username who is the owner of a file/directory. |
| Type | String |
| Default Value | <empty> (means keeping it unchanged) |
| Valid Values | Any valid username. |
| Syntax | Any string. |
+ See also:
+ SETOWNER
+
| Name | permission |
| Description | The permission of a file/directory. |
| Type | Octal |
| Default Value | 755 |
| Valid Values | 0 - 1777 |
| Syntax | Any radix-8 integer (leading zeros may be omitted.) |
+ See also:
+ CREATE,
+ MKDIRS,
+ SETPERMISSION
+
| Name | recursive |
| Description | Should the operation act on the content in the subdirectories? |
| Type | boolean |
| Default Value | false |
| Valid Values | true | false |
| Syntax | true | false |
+ See also:
+ RENAME
+
| Name | renewer |
| Description | The username of the renewer of a delegation token. |
| Type | String |
| Default Value | <empty> (means the current user) |
| Valid Values | Any valid username. |
| Syntax | Any string. |
+ See also:
+ GETDELEGATIONTOKEN
+
| Name | replication |
| Description | The number of replications of a file. |
| Type | short |
| Default Value | Specified in the configuration. |
| Valid Values | > 0 |
| Syntax | Any integer. |
+ See also:
+ CREATE,
+ SETREPLICATION
+
| Name | token |
| Description | The delegation token used for the operation. |
| Type | String |
| Default Value | <empty> |
| Valid Values | An encoded token. |
| Syntax | See the note in Delegation. |
+ See also:
+ RENEWDELEGATIONTOKEN,
+ CANCELDELEGATIONTOKEN
+
| Name | user.name |
| Description | The authenticated user; see Authentication. |
| Type | String |
| Default Value | null |
| Valid Values | Any valid username. |
| Syntax | Any string. |
+ See also: + Authentication +
+