big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge trunk into HA branch

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1170378 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Aaron Myers 2011-09-13 22:49:27 +00:00
commit f4b2b63807
340 changed files with 13936 additions and 4009 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
dev-support/smart-apply-patch.sh
View File

@ -39,21 +39,40 @@ fi
# Come up with a list of changed files into $TMP
TMP=/tmp/tmp.paths.$$
TOCLEAN="$TOCLEAN $TMP"
grep '^+++\|^---' $PATCH_FILE | cut -c '5-' | grep -v /dev/null | sort | uniq > $TMP
# Assume p0 to start
if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
PLEVEL=0
#if the patch applied at P0 there is the possability that all we are doing
# is adding new files and they would apply anywhere. So try to guess the
# correct place to put those files.
TMP2=/tmp/tmp.paths.2.$$
TOCLEAN="$TOCLEAN $TMP2"
grep '^patching file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2
#first off check that all of the files do not exist
FOUND_ANY=0
for CHECK_FILE in $(cat $TMP2)
do
if [[ -f $CHECK_FILE ]]; then
FOUND_ANY=1
fi
done
if [[ "$FOUND_ANY" = "0" ]]; then
#all of the files are new files so we have to guess where the correct place to put it is.
# if all of the lines start with a/ or b/, then this is a git patch that
# was generated without --no-prefix
if ! grep -qv '^a/\|^b/' $TMP ; then
if ! grep -qv '^a/\|^b/' $TMP2 ; then
echo Looks like this is a git patch. Stripping a/ and b/ prefixes
echo and incrementing PLEVEL
PLEVEL=$[$PLEVEL + 1]
sed -i -e 's,^[ab]/,,' $TMP
sed -i -e 's,^[ab]/,,' $TMP2
fi
PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 | sort | uniq)
# if we are at the project root then nothing more to do
if [[ -d hadoop-common-project ]]; then
@ -62,17 +81,26 @@ if [[ -d hadoop-common-project ]]; then
# if all of the lines start with hadoop-common/, hadoop-hdfs/, or hadoop-mapreduce/, this is
# relative to the hadoop root instead of the subproject root, so we need
# to chop off another layer
elif [[ "$PREFIX_DIRS" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$ ]]; then
elif [[ "$PREFIX_DIRS_AND_FILES" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$ ]]; then
echo Looks like this is relative to project root. Increasing PLEVEL
PLEVEL=$[$PLEVEL + 1]
elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project' ; then
elif ! echo "$PREFIX_DIRS_AND_FILES" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project' ; then
echo Looks like this is a cross-subproject patch. Try applying from the project root
exit 1
cleanup 1
fi
fi
elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
PLEVEL=1
elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
PLEVEL=2
else
echo "The patch does not appear to apply with p0 to p2";
cleanup 1;
fi
echo Going to apply patch with: $PATCH -p$PLEVEL
$PATCH -p$PLEVEL -E < $PATCH_FILE
cleanup 0
cleanup $?

70
dev-support/test-patch.sh
View File

@ -64,6 +64,7 @@ printUsage() {
echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
echo "--forrest-home=<path> Forrest home directory (default FORREST_HOME environment variable)"
echo "--dirty-workspace Allow the local SVN workspace to have uncommitted changes"
echo "--run-tests Run all tests below the base directory"
echo
echo "Jenkins-only options:"
echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)"
@ -130,6 +131,9 @@ parseArgs() {
--dirty-workspace)
DIRTY_WORKSPACE=true
;;
--run-tests)
RUN_TESTS=true
;;
*)
PATCH_OR_DEFECT=$i
;;
@ -249,6 +253,18 @@ setup () {
echo "======================================================================"
echo ""
echo ""
if [[ ! -d hadoop-common-project ]]; then
cd $bindir/..
echo "Compiling $(pwd)"
echo "$MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1"
$MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1
if [[ $? != 0 ]] ; then
echo "Top-level trunk compilation is broken?"
cleanupAndExit 1
fi
cd -
fi
echo "Compiling $(pwd)"
echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
if [[ $? != 0 ]] ; then
@ -550,8 +566,10 @@ checkFindbugsWarnings () {
$FINDBUGS_HOME/bin/convertXmlToText -html \
$PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
$PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
if [[ $newFindbugsWarnings > 0 ]] ; then
JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/$(basename $BASEDIR)/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
$JIRA_COMMENT_FOOTER"
fi
done
### if current warnings greater than OK_FINDBUGS_WARNINGS
@ -580,26 +598,12 @@ runTests () {
echo ""
echo ""
failed_tests=""
modules=$(findModules)
for module in $modules;
do
pushd $module
echo " Running tests in $module"
### Kill any rogue build processes from the last attempt
$PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
if [[ $? != 0 ]] ; then
### Find and format names of failed tests
module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"`
failed_tests="${failed_tests}
${module_failed_tests}"
failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"`
fi
popd
done
echo $failed_tests
if [[ -n "$failed_tests" ]] ; then
@ -615,36 +619,6 @@ $failed_tests"
return 0
}
###############################################################################
### Find the modules changed by the patch
findModules () {
# Come up with a list of changed files into $TMP
TMP=/tmp/tmp.paths.$$
$GREP '^+++\|^---' $PATCH_DIR/patch | cut -c '5-' | $GREP -v /dev/null | sort | uniq > $TMP
# if all of the lines start with a/ or b/, then this is a git patch that
# was generated without --no-prefix
if ! $GREP -qv '^a/\|^b/' $TMP ; then
sed -i -e 's,^[ab]/,,' $TMP
fi
PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
# if all of the lines start with hadoop-common-project/, hadoop-hdfs-project/, or hadoop-mapreduce-project/, this is
# relative to the hadoop root instead of the subproject root
if [[ "$PREFIX_DIRS" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$ ]]; then
echo $PREFIX_DIRS
return 0
elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project' ; then
echo $PREFIX_DIRS
return 0
fi
# No modules found. Running from current directory.
echo .
}
###############################################################################
### Run the test-contrib target
runContribTests () {
@ -820,8 +794,8 @@ checkFindbugsWarnings
(( RESULT = RESULT + $? ))
checkReleaseAuditWarnings
(( RESULT = RESULT + $? ))
### Do not call these when run by a developer
if [[ $JENKINS == "true" ]] ; then
### Run tests for Jenkins or if explictly asked for by a developer
if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
runTests
(( RESULT = RESULT + $? ))
runContribTests

7
hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
View File

@ -78,6 +78,13 @@
<include>*-site.xml</include>
</includes>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/packages/templates/conf</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/templates/conf</outputDirectory>
<includes>
<include>*</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.build.directory}</directory>
<outputDirectory>/share/hadoop/${hadoop.component}</outputDirectory>

25
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -5,13 +5,19 @@ Trunk (unreleased changes)
IMPROVEMENTS
HADOOP-7595. Upgrade dependency to Avro 1.5.3. (Alejandro Abdelnur via atm)
HADOOP-7524 Change RPC to allow multiple protocols including multuple versions of the same protocol (sanjay Radia)
HADOOP-7524. Change RPC to allow multiple protocols including multuple
versions of the same protocol (sanjay Radia)
HADOOP-7607. Simplify the RPC proxy cleanup process. (atm)
BUGS
HADOOP-7606. Upgrade Jackson to version 1.7.1 to match the version required
by Jersey (Alejandro Abdelnur via atm)
HADOOP-7610. Fix for hadoop debian package (Eric Yang via gkesavan)
Release 0.23.0 - Unreleased
INCOMPATIBLE CHANGES
@ -371,6 +377,12 @@ Release 0.23.0 - Unreleased
HADOOP-7507. Allow ganglia metrics to include the metrics system tags
in the gmetric names. (Alejandro Abdelnur via todd)
HADOOP-7612. Change test-patch to run tests for all nested modules.
(tomwhite)
HADOOP-7599. Script improvements to setup a secure Hadoop cluster
(Eric Yang via ddas)
OPTIMIZATIONS
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole
@ -563,6 +575,14 @@ Release 0.23.0 - Unreleased
HADOOP-7593. Fix AssertionError in TestHttpServer.testMaxThreads().
(Uma Maheswara Rao G via szetszwo)
HADOOP-7598. Fix smart-apply-patch.sh to handle patching from a sub
directory correctly. (Robert Evans via acmurthy)
HADOOP-7328. When a serializer class is missing, return null, not throw
an NPE. (Harsh J Chouraria via todd)
HADOOP-7626. Bugfix for a config generator (Eric Yang via ddas)
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
@ -1075,6 +1095,9 @@ Release 0.22.0 - Unreleased
HADOOP-7390. VersionInfo not generated properly in git after unsplit. (todd
via atm)
HADOOP-7568. SequenceFile should not print into stdout.
(Plamen Jeliazkov via shv)
Release 0.21.1 - Unreleased
IMPROVEMENTS

1
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
View File

@ -1668,7 +1668,6 @@ private void initialize(Path filename, FSDataInputStream in,
try {
seek(start);
this.end = this.in.getPos() + length;
System.out.println("Setting end to " + end);
// if it wrapped around, use the max
if (end < length) {
end = Long.MAX_VALUE;

17
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
View File

@ -27,10 +27,10 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.serializer.avro.AvroReflectSerialization;
import org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
/**
* <p>
@ -55,7 +55,8 @@ public class SerializationFactory extends Configured {
*/
public SerializationFactory(Configuration conf) {
super(conf);
for (String serializerName : conf.getStrings("io.serializations",
for (String serializerName : conf.getStrings(
CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
new String[]{WritableSerialization.class.getName(),
AvroSpecificSerialization.class.getName(),
AvroReflectSerialization.class.getName()})) {
@ -76,11 +77,19 @@ private void add(Configuration conf, String serializationName) {
}
public <T> Serializer<T> getSerializer(Class<T> c) {
return getSerialization(c).getSerializer(c);
Serialization<T> serializer = getSerialization(c);
if (serializer != null) {
return serializer.getSerializer(c);
}
return null;
}
public <T> Deserializer<T> getDeserializer(Class<T> c) {
return getSerialization(c).getDeserializer(c);
Serialization<T> serializer = getSerialization(c);
if (serializer != null) {
return serializer.getDeserializer(c);
}
return null;
}
@SuppressWarnings("unchecked")

11
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AvroRpcEngine.java
View File

@ -131,7 +131,7 @@ public void writeBuffers(List<ByteBuffer> buffers) throws IOException {
}
public void close() throws IOException {
ENGINE.stopProxy(tunnel);
RPC.stopProxy(tunnel);
}
}
@ -152,15 +152,6 @@ public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
false);
}
/** Stop this proxy. */
public void stopProxy(Object proxy) {
try {
((Invoker)Proxy.getInvocationHandler(proxy)).close();
} catch (IOException e) {
LOG.warn("Error while stopping "+proxy, e);
}
}
private class Invoker implements InvocationHandler, Closeable {
private final ClientTransceiver tx;
private final SpecificRequestor requestor;

54
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.ipc;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.lang.reflect.Method;
@ -26,6 +27,7 @@
import java.net.NoRouteToHostException;
import java.net.SocketTimeoutException;
import java.io.*;
import java.io.Closeable;
import java.util.Map;
import java.util.HashMap;
@ -80,12 +82,8 @@ static public String getProtocolName(Class<?> protocol) {
private RPC() {} // no public ctor
// cache of RpcEngines by protocol
private static final Map<Class,RpcEngine> PROTOCOL_ENGINES
= new HashMap<Class,RpcEngine>();
// track what RpcEngine is used by a proxy class, for stopProxy()
private static final Map<Class,RpcEngine> PROXY_ENGINES
= new HashMap<Class,RpcEngine>();
private static final Map<Class<?>,RpcEngine> PROTOCOL_ENGINES
= new HashMap<Class<?>,RpcEngine>();
private static final String ENGINE_PROP = "rpc.engine";
@ -96,32 +94,23 @@ private RPC() {} // no public ctor
* @param engine the RpcEngine impl
*/
public static void setProtocolEngine(Configuration conf,
Class protocol, Class engine) {
Class<?> protocol, Class<?> engine) {
conf.setClass(ENGINE_PROP+"."+protocol.getName(), engine, RpcEngine.class);
}
// return the RpcEngine configured to handle a protocol
private static synchronized RpcEngine getProtocolEngine(Class protocol,
private static synchronized RpcEngine getProtocolEngine(Class<?> protocol,
Configuration conf) {
RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
if (engine == null) {
Class<?> impl = conf.getClass(ENGINE_PROP+"."+protocol.getName(),
WritableRpcEngine.class);
engine = (RpcEngine)ReflectionUtils.newInstance(impl, conf);
if (protocol.isInterface())
PROXY_ENGINES.put(Proxy.getProxyClass(protocol.getClassLoader(),
protocol),
engine);
PROTOCOL_ENGINES.put(protocol, engine);
}
return engine;
}
// return the RpcEngine that handles a proxy object
private static synchronized RpcEngine getProxyEngine(Object proxy) {
return PROXY_ENGINES.get(proxy.getClass());
}
/**
* A version mismatch for the RPC protocol.
*/
@ -477,13 +466,30 @@ public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
}
/**
* Stop this proxy and release its invoker's resource
* @param proxy the proxy to be stopped
* Stop this proxy and release its invoker's resource by getting the
* invocation handler for the given proxy object and calling
* {@link Closeable#close} if that invocation handler implements
* {@link Closeable}.
*
* @param proxy the RPC proxy object to be stopped
*/
public static void stopProxy(Object proxy) {
RpcEngine rpcEngine;
if (proxy!=null && (rpcEngine = getProxyEngine(proxy)) != null) {
rpcEngine.stopProxy(proxy);
InvocationHandler invocationHandler = null;
try {
invocationHandler = Proxy.getInvocationHandler(proxy);
} catch (IllegalArgumentException e) {
LOG.error("Tried to call RPC.stopProxy on an object that is not a proxy.", e);
}
if (proxy != null && invocationHandler != null &&
invocationHandler instanceof Closeable) {
try {
((Closeable)invocationHandler).close();
} catch (IOException e) {
LOG.error("Stopping RPC invocation handler caused exception", e);
}
} else {
LOG.error("Could not get invocation handler " + invocationHandler +
" for proxy " + proxy + ", or invocation handler is not closeable.");
}
}
@ -532,7 +538,7 @@ public static Server getServer(final Object instance, final String bindAddress,
}
/** Construct a server for a protocol implementation instance. */
public static Server getServer(Class protocol,
public static Server getServer(Class<?> protocol,
Object instance, String bindAddress,
int port, Configuration conf)
throws IOException {
@ -543,7 +549,7 @@ public static Server getServer(Class protocol,
* @deprecated secretManager should be passed.
*/
@Deprecated
public static Server getServer(Class protocol,
public static Server getServer(Class<?> protocol,
Object instance, String bindAddress, int port,
int numHandlers,
boolean verbose, Configuration conf)

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
View File

@ -41,9 +41,6 @@ <T> ProtocolProxy<T> getProxy(Class<T> protocol,
UserGroupInformation ticket, Configuration conf,
SocketFactory factory, int rpcTimeout) throws IOException;
/** Stop this proxy. */
void stopProxy(Object proxy);
/** Expert: Make multiple, parallel calls to a set of servers. */
Object[] call(Method method, Object[][] params, InetSocketAddress[] addrs,
UserGroupInformation ticket, Configuration conf)

14
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
View File

@ -30,6 +30,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.io.Closeable;
import java.util.Map;
import java.util.HashMap;
@ -219,7 +220,7 @@ public Configuration getConf() {
private static ClientCache CLIENTS=new ClientCache();
private static class Invoker implements InvocationHandler {
private static class Invoker implements InvocationHandler, Closeable {
private Client.ConnectionId remoteId;
private Client client;
private boolean isClosed = false;
@ -250,7 +251,7 @@ public Object invoke(Object proxy, Method method, Object[] args)
}
/* close the IPC client that's responsible for this invoker's RPCs */
synchronized private void close() {
synchronized public void close() {
if (!isClosed) {
isClosed = true;
CLIENTS.stopClient(client);
@ -282,15 +283,6 @@ public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
return new ProtocolProxy<T>(protocol, proxy, true);
}
/**
* Stop this proxy and release its invoker's resource
* @param proxy the proxy to be stopped
*/
public void stopProxy(Object proxy) {
((Invoker)Proxy.getInvocationHandler(proxy)).close();
}
/** Expert: Make multiple, parallel calls to a set of servers. */
public Object[] call(Method method, Object[][] params,
InetSocketAddress[] addrs,

23
hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-datanode
View File

@ -39,6 +39,14 @@ fi
. /lib/lsb/init-functions
if [ -n "$HADOOP_SECURE_DN_USER" ]; then
DN_USER="root"
IDENT_USER=${HADOOP_SECURE_DN_USER}
else
DN_USER="hdfs"
IDENT_USER=${DN_USER}
fi
# Are we running from init?
run_by_init() {
([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
@ -67,13 +75,14 @@ check_privsep_dir() {
}
export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
export HADOOP_PREFIX="/usr"
case "$1" in
start)
check_privsep_dir
check_for_no_start
log_daemon_msg "Starting Apache Hadoop Data Node server" "hadoop-datanode"
if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
log_end_msg 0
else
log_end_msg 1
@ -81,7 +90,7 @@ case "$1" in
;;
stop)
log_daemon_msg "Stopping Apache Hadoop Data Node server" "hadoop-datanode"
if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid; then
if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid; then
log_end_msg 0
else
log_end_msg 1
@ -91,9 +100,9 @@ case "$1" in
restart)
check_privsep_dir
log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid
start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid
check_for_no_start log_end_msg
if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
log_end_msg 0
else
log_end_msg 1
@ -104,14 +113,14 @@ case "$1" in
check_privsep_dir
log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
set +e
start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid
start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid
RET="$?"
set -e
case $RET in
0)
# old daemon stopped
check_for_no_start log_end_msg
if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
log_end_msg 0
else
log_end_msg 1
@ -131,7 +140,7 @@ case "$1" in
;;
status)
status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid ${JAVA_HOME}/bin/java hadoop-datanode && exit 0 || exit $?
status_of_proc -p ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid ${JAVA_HOME}/bin/java hadoop-datanode && exit 0 || exit $?
;;
*)

1
hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-jobtracker
View File

@ -67,6 +67,7 @@ check_privsep_dir() {
}
export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
export HADOOP_PREFIX="/usr"
case "$1" in
start)

3
hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-namenode
View File

@ -67,10 +67,11 @@ check_privsep_dir() {
}
format() {
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format' hdfs
sudo -u hdfs ${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format
}
export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
export HADOOP_PREFIX="/usr"
case "$1" in
start)

1
hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-tasktracker
View File

@ -67,6 +67,7 @@ check_privsep_dir() {
}
export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
export HADOOP_PREFIX="/usr"
case "$1" in
start)

93
hadoop-common-project/hadoop-common/src/main/packages/hadoop-create-user.sh
View File

@ -14,9 +14,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
script="$(basename -- "$this")"
this="$bin/$script"
if [ "$HADOOP_HOME" != "" ]; then
echo "Warning: \$HADOOP_HOME is deprecated."
@ -29,30 +30,86 @@ usage() {
echo "
usage: $0 <parameters>
Require parameter:
--config /etc/hadoop Location of Hadoop configuration file
-u <username> Create user on HDFS
Optional parameters:
-h Display this message
--kerberos-realm=KERBEROS.EXAMPLE.COM Set Kerberos realm
--super-user=hdfs Set super user id
--super-user-keytab=/etc/security/keytabs/hdfs.keytab Set super user keytab location
"
exit 1
}
# Parse script parameters
if [ $# != 2 ] ; then
OPTS=$(getopt \
-n $0 \
-o '' \
-l 'kerberos-realm:' \
-l 'super-user:' \
-l 'super-user-keytab:' \
-o 'h' \
-o 'u' \
-- "$@")
if [ $? != 0 ] ; then
usage
exit 1
fi
while getopts "hu:" OPTION
do
case $OPTION in
u)
SETUP_USER=$2; shift 2
create_user() {
if [ "${SETUP_USER}" = "" ]; then
break
fi
HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
export HADOOP_PREFIX
export HADOOP_CONF_DIR
export JAVA_HOME
export SETUP_USER=${SETUP_USER}
export SETUP_PATH=/user/${SETUP_USER}
if [ ! "${KERBEROS_REALM}" = "" ]; then
# locate kinit cmd
if [ -e /etc/lsb-release ]; then
KINIT_CMD="/usr/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
else
KINIT_CMD="/usr/kerberos/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
fi
su -c "${KINIT_CMD}" ${HADOOP_HDFS_USER}
fi
su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir ${SETUP_PATH}" ${HADOOP_HDFS_USER}
su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown ${SETUP_USER}:${SETUP_USER} ${SETUP_PATH}" ${HADOOP_HDFS_USER}
su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 711 ${SETUP_PATH}" ${HADOOP_HDFS_USER}
if [ "$?" == "0" ]; then
echo "User directory has been setup: ${SETUP_PATH}"
fi
}
eval set -- "${OPTS}"
while true; do
case "$1" in
-u)
shift
;;
h)
--kerberos-realm)
KERBEROS_REALM=$2; shift 2
;;
--super-user)
HADOOP_HDFS_USER=$2; shift 2
;;
--super-user-keytab)
HDFS_USER_KEYTAB=$2; shift 2
;;
-h)
usage
;;
--)
shift ; break
while shift; do
SETUP_USER=$1
create_user
done
break
;;
*)
echo "Unknown option: $1"
@ -62,15 +119,3 @@ do
esac
done
# Create user directory on HDFS
export SETUP_USER
export SETUP_PATH=/user/${SETUP_USER}
export HADOOP_PREFIX
export HADOOP_CONF_DIR
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir ${SETUP_PATH}' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown ${SETUP_USER}:${SETUP_USER} ${SETUP_PATH}' hdfs
if [ "$?" == "0" ]; then
echo "User directory has been setup: ${SETUP_PATH}"
fi

374
hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
View File

@ -18,35 +18,60 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
if [ "$HADOOP_HOME" != "" ]; then
echo "Warning: \$HADOOP_HOME is deprecated."
echo
fi
. "$bin"/../libexec/hadoop-config.sh
this="${BASH_SOURCE-$0}"
export HADOOP_PREFIX=`dirname "$this"`/..
usage() {
echo "
usage: $0 <parameters>
Optional parameters:
--auto Setup automatically
--default Generate default config
--conf-dir=/etc/hadoop Set config directory
--auto Setup path and configuration automatically
--default Setup configuration as default
--conf-dir=/etc/hadoop Set configuration directory
--datanode-dir=/var/lib/hadoop/hdfs/datanode Set datanode directory
--group=hadoop Set Hadoop group name
-h Display this message
--jobtracker-url=hostname:9001 Set jobtracker url
--hdfs-user=hdfs Set HDFS user
--jobtracker-host=hostname Set jobtracker host
--namenode-host=hostname Set namenode host
--secondarynamenode-host=hostname Set secondary namenode host
--kerberos-realm=KERBEROS.EXAMPLE.COM Set Kerberos realm
--kinit-location=/usr/kerberos/bin/kinit Set kinit location
--keytab-dir=/etc/security/keytabs Set keytab directory
--log-dir=/var/log/hadoop Set log directory
--hdfs-dir=/var/lib/hadoop/hdfs Set hdfs directory
--pid-dir=/var/run/hadoop Set pid directory
--hdfs-dir=/var/lib/hadoop/hdfs Set HDFS directory
--hdfs-user-keytab=/home/hdfs/hdfs.keytab Set HDFS user key tab
--mapred-dir=/var/lib/hadoop/mapred Set mapreduce directory
--mapreduce-user=mr Set mapreduce user
--mapreduce-user-keytab=/home/mr/hdfs.keytab Set mapreduce user key tab
--namenode-dir=/var/lib/hadoop/hdfs/namenode Set namenode directory
--namenode-url=hdfs://hostname:9000/ Set namenode url
--replication=3 Set replication factor
--taskscheduler=org.apache.hadoop.mapred.JobQueueTaskScheduler Set task scheduler
--datanodes=hostname1,hostname2,... SET the datanodes
--tasktrackers=hostname1,hostname2,... SET the tasktrackers
"
exit 1
}
check_permission() {
TARGET=$1
OWNER="0"
RESULT=0
while [ "$TARGET" != "/" ]; do
PARENT=`dirname $TARGET`
NAME=`basename $TARGET`
OWNER=`ls -ln $PARENT | grep $NAME| awk '{print $3}'`
if [ "$OWNER" != "0" ]; then
RESULT=1
break
fi
TARGET=`dirname $TARGET`
done
return $RESULT
}
template_generator() {
REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
cat $1 |
@ -64,17 +89,30 @@ OPTS=$(getopt \
-n $0 \
-o '' \
-l 'auto' \
-l 'java-home:' \
-l 'conf-dir:' \
-l 'default' \
-l 'group:' \
-l 'hdfs-dir:' \
-l 'namenode-dir:' \
-l 'datanode-dir:' \
-l 'mapred-dir:' \
-l 'namenode-url:' \
-l 'jobtracker-url:' \
-l 'namenode-host:' \
-l 'secondarynamenode-host:' \
-l 'jobtracker-host:' \
-l 'log-dir:' \
-l 'pid-dir:' \
-l 'replication:' \
-l 'taskscheduler:' \
-l 'hdfs-user:' \
-l 'hdfs-user-keytab:' \
-l 'mapreduce-user:' \
-l 'mapreduce-user-keytab:' \
-l 'keytab-dir:' \
-l 'kerberos-realm:' \
-l 'kinit-location:' \
-l 'datanodes:' \
-l 'tasktrackers:' \
-o 'h' \
-- "$@")
@ -85,7 +123,6 @@ fi
# Make sure the HADOOP_LOG_DIR is not picked up from user environment.
unset HADOOP_LOG_DIR
# Parse script parameters
eval set -- "${OPTS}"
while true ; do
case "$1" in
@ -94,6 +131,10 @@ while true ; do
AUTOMATED=1
shift
;;
--java-home)
JAVA_HOME=$2; shift 2
AUTOMATED=1
;;
--conf-dir)
HADOOP_CONF_DIR=$2; shift 2
AUTOMATED=1
@ -101,6 +142,10 @@ while true ; do
--default)
AUTOMATED=1; shift
;;
--group)
HADOOP_GROUP=$2; shift 2
AUTOMATED=1
;;
-h)
usage
;;
@ -120,11 +165,15 @@ while true ; do
HADOOP_MAPRED_DIR=$2; shift 2
AUTOMATED=1
;;
--namenode-url)
--namenode-host)
HADOOP_NN_HOST=$2; shift 2
AUTOMATED=1
;;
--jobtracker-url)
--secondarynamenode-host)
HADOOP_SNN_HOST=$2; shift 2
AUTOMATED=1
;;
--jobtracker-host)
HADOOP_JT_HOST=$2; shift 2
AUTOMATED=1
;;
@ -132,6 +181,10 @@ while true ; do
HADOOP_LOG_DIR=$2; shift 2
AUTOMATED=1
;;
--pid-dir)
HADOOP_PID_DIR=$2; shift 2
AUTOMATED=1
;;
--replication)
HADOOP_REPLICATION=$2; shift 2
AUTOMATED=1
@ -140,6 +193,45 @@ while true ; do
HADOOP_TASK_SCHEDULER=$2; shift 2
AUTOMATED=1
;;
--hdfs-user)
HADOOP_HDFS_USER=$2; shift 2
AUTOMATED=1
;;
--mapreduce-user)
HADOOP_MR_USER=$2; shift 2
AUTOMATED=1
;;
--keytab-dir)
KEYTAB_DIR=$2; shift 2
AUTOMATED=1
;;
--hdfs-user-keytab)
HDFS_KEYTAB=$2; shift 2
AUTOMATED=1
;;
--mapreduce-user-keytab)
MR_KEYTAB=$2; shift 2
AUTOMATED=1
;;
--kerberos-realm)
KERBEROS_REALM=$2; shift 2
SECURITY_TYPE="kerberos"
AUTOMATED=1
;;
--kinit-location)
KINIT=$2; shift 2
AUTOMATED=1
;;
--datanodes)
DATANODES=$2; shift 2
AUTOMATED=1
DATANODES=$(echo $DATANODES | tr ',' ' ')
;;
--tasktrackers)
TASKTRACKERS=$2; shift 2
AUTOMATED=1
TASKTRACKERS=$(echo $TASKTRACKERS | tr ',' ' ')
;;
--)
shift ; break
;;
@ -151,21 +243,40 @@ while true ; do
esac
done
# Fill in default values, if parameters have not been defined.
AUTOSETUP=${AUTOSETUP:-1}
JAVA_HOME=${JAVA_HOME:-/usr/java/default}
HADOOP_NN_HOST=${HADOOP_NN_HOST:-hdfs://`hostname`:9000/}
HADOOP_GROUP=${HADOOP_GROUP:-hadoop}
HADOOP_NN_HOST=${HADOOP_NN_HOST:-`hostname`}
HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
HADOOP_JT_HOST=${HADOOP_JT_HOST:-`hostname`:9001}
HADOOP_JT_HOST=${HADOOP_JT_HOST:-`hostname`}
HADOOP_HDFS_DIR=${HADOOP_HDFS_DIR:-/var/lib/hadoop/hdfs}
HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-/var/log/hadoop}
HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/log/hadoop}
HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
HADOOP_REPLICATION=${HADOOP_RELICATION:-3}
HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}
KERBEROS_REALM=${KERBEROS_REALM:-KERBEROS.EXAMPLE.COM}
SECURITY_TYPE=${SECURITY_TYPE:-simple}
KINIT=${KINIT:-/usr/kerberos/bin/kinit}
if [ "${SECURITY_TYPE}" = "kerberos" ]; then
TASK_CONTROLLER="org.apache.hadoop.mapred.LinuxTaskController"
HADOOP_DN_ADDR="0.0.0.0:1019"
HADOOP_DN_HTTP_ADDR="0.0.0.0:1022"
SECURITY="true"
else
TASK_CONTROLLER="org.apache.hadoop.mapred.DefaultTaskController"
HADDOP_DN_ADDR="0.0.0.0:50010"
HADOOP_DN_HTTP_ADDR="0.0.0.0:50075"
SECURITY="false"
fi
# Interactive setup wizard
if [ "${AUTOMATED}" != "1" ]; then
echo "Setup Hadoop Configuration"
echo
@ -173,18 +284,18 @@ if [ "${AUTOMATED}" != "1" ]; then
read USER_HADOOP_CONF_DIR
echo -n "Where would you like to put log directory? (${HADOOP_LOG_DIR}) "
read USER_HADOOP_LOG_DIR
echo -n "What is the url of the namenode? (${HADOOP_NN_HOST}) "
echo -n "Where would you like to put pid directory? (${HADOOP_PID_DIR}) "
read USER_HADOOP_PID_DIR
echo -n "What is the host of the namenode? (${HADOOP_NN_HOST}) "
read USER_HADOOP_NN_HOST
echo -n "Where would you like to put namenode data directory? (${HADOOP_NN_DIR}) "
read USER_HADOOP_NN_DIR
echo -n "Where would you like to put datanode data directory? (${HADOOP_DN_DIR}) "
read USER_HADOOP_DN_DIR
echo -n "What is the url of the jobtracker? (${HADOOP_JT_HOST}) "
echo -n "What is the host of the jobtracker? (${HADOOP_JT_HOST}) "
read USER_HADOOP_JT_HOST
echo -n "Where would you like to put jobtracker/tasktracker data directory? (${HADOOP_MAPRED_DIR}) "
read USER_HADOOP_MAPRED_DIR
echo -n "Which taskscheduler would you like? (${HADOOP_TASK_SCHEDULER}) "
read USER_HADOOP_TASK_SCHEDULER
echo -n "Where is JAVA_HOME directory? (${JAVA_HOME}) "
read USER_JAVA_HOME
echo -n "Would you like to create directories/copy conf files to localhost? (Y/n) "
@ -199,16 +310,18 @@ if [ "${AUTOMATED}" != "1" ]; then
HADOOP_MAPRED_DIR=${USER_HADOOP_MAPRED_DIR:-$HADOOP_MAPRED_DIR}
HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
HADOOP_LOG_DIR=${USER_HADOOP_LOG_DIR:-$HADOOP_LOG_DIR}
HADOOP_PID_DIR=${USER_HADOOP_PID_DIR:-$HADOOP_PID_DIR}
HADOOP_CONF_DIR=${USER_HADOOP_CONF_DIR:-$HADOOP_CONF_DIR}
AUTOSETUP=${USER_AUTOSETUP:-y}
echo "Review your choices:"
echo
echo "Config directory : ${HADOOP_CONF_DIR}"
echo "Log directory : ${HADOOP_LOG_DIR}"
echo "Namenode url : ${HADOOP_NN_HOST}"
echo "PID directory : ${HADOOP_PID_DIR}"
echo "Namenode host : ${HADOOP_NN_HOST}"
echo "Namenode directory : ${HADOOP_NN_DIR}"
echo "Datanode directory : ${HADOOP_DN_DIR}"
echo "Jobtracker url : ${HADOOP_JT_HOST}"
echo "Jobtracker host : ${HADOOP_JT_HOST}"
echo "Mapreduce directory : ${HADOOP_MAPRED_DIR}"
echo "Task scheduler : ${HADOOP_TASK_SCHEDULER}"
echo "JAVA_HOME directory : ${JAVA_HOME}"
@ -222,61 +335,180 @@ if [ "${AUTOMATED}" != "1" ]; then
fi
fi
if [ "${AUTOSETUP}" == "1" ]; then
# If user wants to setup local system automatically,
# set config file generation location to HADOOP_CONF_DIR.
DEST=${HADOOP_CONF_DIR}
else
# If user is only interested to generate config file locally,
# place config files in the current working directory.
DEST=`pwd`
if [ "${AUTOSETUP}" == "1" -o "${AUTOSETUP}" == "y" ]; then
if [ -d ${KEYTAB_DIR} ]; then
chmod 700 ${KEYTAB_DIR}/*
chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${KEYTAB_DIR}/[jt]t.service.keytab
chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${KEYTAB_DIR}/[dns]n.service.keytab
fi
chmod 755 -R ${HADOOP_PREFIX}/sbin/*hadoop*
chmod 755 -R ${HADOOP_PREFIX}/bin/hadoop
chmod 755 -R ${HADOOP_PREFIX}/libexec/hadoop-config.sh
mkdir -p /home/${HADOOP_MR_USER}
chown ${HADOOP_MR_USER}:${HADOOP_GROUP} /home/${HADOOP_MR_USER}
HDFS_DIR=`echo ${HADOOP_HDFS_DIR} | sed -e 's/,/ /g'`
mkdir -p ${HDFS_DIR}
if [ -e ${HADOOP_NN_DIR} ]; then
rm -rf ${HADOOP_NN_DIR}
fi
DATANODE_DIR=`echo ${HADOOP_DN_DIR} | sed -e 's/,/ /g'`
mkdir -p ${DATANODE_DIR}
MAPRED_DIR=`echo ${HADOOP_MAPRED_DIR} | sed -e 's/,/ /g'`
mkdir -p ${MAPRED_DIR}
mkdir -p ${HADOOP_CONF_DIR}
check_permission ${HADOOP_CONF_DIR}
if [ $? == 1 ]; then
echo "Full path to ${HADOOP_CONF_DIR} should be owned by root."
exit 1
fi
# remove existing config file, they are existed in current directory.
rm -f ${DEST}/core-site.xml >/dev/null
rm -f ${DEST}/hdfs-site.xml >/dev/null
rm -f ${DEST}/mapred-site.xml >/dev/null
rm -f ${DEST}/hadoop-env.sh >/dev/null
# Generate config file with specified parameters.
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/core-site.xml ${DEST}/core-site.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/hdfs/templates/hdfs-site.xml ${DEST}/hdfs-site.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/mapreduce/templates/mapred-site.xml ${DEST}/mapred-site.xml
template_generator ${HADOOP_CONF_DIR}/hadoop-env.sh.template ${DEST}/hadoop-env.sh
chown root:hadoop ${DEST}/hadoop-env.sh
chmod 755 ${DEST}/hadoop-env.sh
# Setup directory path and copy config files, if AUTOSETUP is chosen.
if [ "${AUTOSETUP}" == "1" -o "${AUTOSETUP}" == "y" ]; then
mkdir -p ${HADOOP_HDFS_DIR}
mkdir -p ${HADOOP_NN_DIR}
mkdir -p ${HADOOP_DN_DIR}
mkdir -p ${HADOOP_MAPRED_DIR}
mkdir -p ${HADOOP_CONF_DIR}
mkdir -p ${HADOOP_LOG_DIR}
mkdir -p ${HADOOP_LOG_DIR}/hdfs
mkdir -p ${HADOOP_LOG_DIR}/mapred
chown hdfs:hadoop ${HADOOP_HDFS_DIR}
chown hdfs:hadoop ${HADOOP_NN_DIR}
chown hdfs:hadoop ${HADOOP_DN_DIR}
chown mapred:hadoop ${HADOOP_MAPRED_DIR}
chown root:hadoop ${HADOOP_LOG_DIR}
#create the log sub dir for diff users
mkdir -p ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
mkdir -p ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
mkdir -p ${HADOOP_PID_DIR}
chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HDFS_DIR}
chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${DATANODE_DIR}
chmod 700 -R ${DATANODE_DIR}
chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${MAPRED_DIR}
chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}
chmod 775 ${HADOOP_LOG_DIR}
chown hdfs:hadoop ${HADOOP_LOG_DIR}/hdfs
chown mapred:hadoop ${HADOOP_LOG_DIR}/mapred
chmod 775 ${HADOOP_PID_DIR}
chown root:${HADOOP_GROUP} ${HADOOP_PID_DIR}
#change the permission and the owner
chmod 755 ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
chmod 755 ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
if [ -e ${HADOOP_CONF_DIR}/core-site.xml ]; then
mv -f ${HADOOP_CONF_DIR}/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml.bak
fi
if [ -e ${HADOOP_CONF_DIR}/hdfs-site.xml ]; then
mv -f ${HADOOP_CONF_DIR}/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml.bak
fi
if [ -e ${HADOOP_CONF_DIR}/mapred-site.xml ]; then
mv -f ${HADOOP_CONF_DIR}/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml.bak
fi
if [ -e ${HADOOP_CONF_DIR}/hadoop-env.sh ]; then
mv -f ${HADOOP_CONF_DIR}/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh.bak
fi
if [ -e ${HADOOP_CONF_DIR}/hadoop-policy.xml ]; then
mv -f ${HADOOP_CONF_DIR}/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml.bak
fi
if [ -e ${HADOOP_CONF_DIR}/mapred-queue-acls.xml ]; then
mv -f ${HADOOP_CONF_DIR}/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml.bak
fi
if [ -e ${HADOOP_CONF_DIR}/commons-logging.properties ]; then
mv -f ${HADOOP_CONF_DIR}/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties.bak
fi
if [ -e ${HADOOP_CONF_DIR}/taskcontroller.cfg ]; then
mv -f ${HADOOP_CONF_DIR}/taskcontroller.cfg ${HADOOP_CONF_DIR}/taskcontroller.cfg.bak
fi
if [ -e ${HADOOP_CONF_DIR}/slaves ]; then
mv -f ${HADOOP_CONF_DIR}/slaves ${HADOOP_CONF_DIR}/slaves.bak
fi
if [ -e ${HADOOP_CONF_DIR}/dfs.include ]; then
mv -f ${HADOOP_CONF_DIR}/dfs.include ${HADOOP_CONF_DIR}/dfs.include.bak
fi
if [ -e ${HADOOP_CONF_DIR}/dfs.exclude ]; then
mv -f ${HADOOP_CONF_DIR}/dfs.exclude ${HADOOP_CONF_DIR}/dfs.exclude.bak
fi
if [ -e ${HADOOP_CONF_DIR}/mapred.include ]; then
mv -f ${HADOOP_CONF_DIR}/mapred.include ${HADOOP_CONF_DIR}/mapred.include.bak
fi
if [ -e ${HADOOP_CONF_DIR}/mapred.exclude ]; then
mv -f ${HADOOP_CONF_DIR}/mapred.exclude ${HADOOP_CONF_DIR}/mapred.exclude.bak
fi
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/taskcontroller.cfg ${HADOOP_CONF_DIR}/taskcontroller.cfg
#set the owner of the hadoop dir to root
chown root ${HADOOP_PREFIX}
chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
#set taskcontroller
chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/taskcontroller.cfg
chmod 400 ${HADOOP_CONF_DIR}/taskcontroller.cfg
chown root:${HADOOP_GROUP} ${HADOOP_PREFIX}/bin/task-controller
chmod 6050 ${HADOOP_PREFIX}/bin/task-controller
#generate the slaves file and include and exclude files for hdfs and mapred
echo '' > ${HADOOP_CONF_DIR}/slaves
echo '' > ${HADOOP_CONF_DIR}/dfs.include
echo '' > ${HADOOP_CONF_DIR}/dfs.exclude
echo '' > ${HADOOP_CONF_DIR}/mapred.include
echo '' > ${HADOOP_CONF_DIR}/mapred.exclude
for dn in $DATANODES
do
echo $dn >> ${HADOOP_CONF_DIR}/slaves
echo $dn >> ${HADOOP_CONF_DIR}/dfs.include
done
for tt in $TASKTRACKERS
do
echo $tt >> ${HADOOP_CONF_DIR}/mapred.include
done
echo "Configuration setup is completed."
if [[ "$HADOOP_NN_HOST" =~ "`hostname`" ]]; then
echo "Proceed to run hadoop-setup-hdfs.sh on namenode."
fi
else
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/taskcontroller.cfg ${HADOOP_CONF_DIR}/taskcontroller.cfg
template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties
if [ ! -e ${HADOOP_CONF_DIR}/capacity-scheduler.xml ]; then
template_generator ${HADOOP_PREFIX}/share/hadoop/templates/conf/capacity-scheduler.xml ${HADOOP_CONF_DIR}/capacity-scheduler.xml
fi
chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
#set taskcontroller
chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/taskcontroller.cfg
chmod 400 ${HADOOP_CONF_DIR}/taskcontroller.cfg
chown root:${HADOOP_GROUP} ${HADOOP_PREFIX}/bin/task-controller
chmod 6050 ${HADOOP_PREFIX}/bin/task-controller
#generate the slaves file and include and exclude files for hdfs and mapred
echo '' > ${HADOOP_CONF_DIR}/slaves
echo '' > ${HADOOP_CONF_DIR}/dfs.include
echo '' > ${HADOOP_CONF_DIR}/dfs.exclude
echo '' > ${HADOOP_CONF_DIR}/mapred.include
echo '' > ${HADOOP_CONF_DIR}/mapred.exclude
for dn in $DATANODES
do
echo $dn >> ${HADOOP_CONF_DIR}/slaves
echo $dn >> ${HADOOP_CONF_DIR}/dfs.include
done
for tt in $TASKTRACKERS
do
echo $tt >> ${HADOOP_CONF_DIR}/mapred.include
done
echo
echo "Configuration file has been generated, please copy:"
echo "Configuration file has been generated in:"
echo
echo "core-site.xml"
echo "hdfs-site.xml"
echo "mapred-site.xml"
echo "hadoop-env.sh"
echo "${HADOOP_CONF_DIR}/core-site.xml"
echo "${HADOOP_CONF_DIR}/hdfs-site.xml"
echo "${HADOOP_CONF_DIR}/mapred-site.xml"
echo "${HADOOP_CONF_DIR}/hadoop-env.sh"
echo
echo " to ${HADOOP_CONF_DIR} on all nodes, and proceed to run hadoop-setup-hdfs.sh on namenode."
fi

110
hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-hdfs.sh
View File

@ -18,37 +18,65 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
if [ "$HADOOP_HOME" != "" ]; then
echo "Warning: \$HADOOP_HOME is deprecated."
echo
fi
. "$bin"/../libexec/hadoop-config.sh
usage() {
echo "
usage: $0 <parameters>
Require parameter:
-c <clusterid> Set cluster identifier for HDFS
Optional parameters:
--format Force namenode format
--group=hadoop Set Hadoop group
-h Display this message
--hdfs-user=hdfs Set HDFS user
--kerberos-realm=KERBEROS.EXAMPLE.COM Set Kerberos realm
--hdfs-user-keytab=/home/hdfs/hdfs.keytab Set HDFS user key tab
--mapreduce-user=mr Set mapreduce user
"
exit 1
}
if [ $# != 2 ] ; then
OPTS=$(getopt \
-n $0 \
-o '' \
-l 'format' \
-l 'hdfs-user:' \
-l 'hdfs-user-keytab:' \
-l 'mapreduce-user:' \
-l 'kerberos-realm:' \
-o 'h' \
-- "$@")
if [ $? != 0 ] ; then
usage
exit 1
fi
while getopts "hc:" OPTION
do
case $OPTION in
c)
SETUP_CLUSTER=$2; shift 2
eval set -- "${OPTS}"
while true ; do
case "$1" in
--format)
FORMAT_NAMENODE=1; shift
AUTOMATED=1
;;
h)
usage
--group)
HADOOP_GROUP=$2; shift 2
AUTOMATED=1
;;
--hdfs-user)
HADOOP_HDFS_USER=$2; shift 2
AUTOMATED=1
;;
--mapreduce-user)
HADOOP_MR_USER=$2; shift 2
AUTOMATED=1
;;
--hdfs-user-keytab)
HDFS_KEYTAB=$2; shift 2
AUTOMATED=1
;;
--kerberos-realm)
KERBEROS_REALM=$2; shift 2
AUTOMATED=1
;;
--)
shift ; break
@ -61,30 +89,56 @@ do
esac
done
export HADOOP_PREFIX
export HADOOP_CONF_DIR
export SETUP_CLUSTER
HADOOP_GROUP=${HADOOP_GROUP:-hadoop}
HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
HADOOP_MAPREDUCE_USER=${HADOOP_MR_USER:-mapred}
if [ "${KERBEROS_REALM}" != "" ]; then
# Determine kerberos location base on Linux distro.
if [ -e /etc/lsb-release ]; then
KERBEROS_BIN=/usr/bin
else
KERBEROS_BIN=/usr/kerberos/bin
fi
kinit_cmd="${KERBEROS_BIN}/kinit -k -t ${HDFS_KEYTAB} ${HADOOP_HDFS_USER}"
su -c "${kinit_cmd}" ${HADOOP_HDFS_USER}
fi
# Start namenode and initialize file system structure
echo "Setup Hadoop Distributed File System"
echo
# Format namenode
if [ "${FORMAT_NAMENODE}" == "1" ]; then
echo "Formatting namenode"
echo
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format -clusterid ${SETUP_CLUSTER}' hdfs
su -c "echo Y | ${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format" ${HADOOP_HDFS_USER}
echo
fi
# Start namenode process
echo "Starting namenode process"
echo
/etc/init.d/hadoop-namenode start
if [ -e ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh ]; then
DAEMON_PATH=${HADOOP_PREFIX}/sbin
else
DAEMON_PATH=${HADOOP_PREFIX}/bin
fi
su -c "${DAEMON_PATH}/hadoop-daemon.sh --config ${HADOOP_CONF_DIR} start namenode" ${HADOOP_HDFS_USER}
echo
echo "Initialize HDFS file system: "
echo
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /jobtracker' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /jobtracker' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /user/mapred' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /user/mapred' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /tmp' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 777 /tmp' hdfs
#create the /user dir
su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /user" ${HADOOP_HDFS_USER}
#create /tmp and give it 777
su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /tmp" ${HADOOP_HDFS_USER}
su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 777 /tmp" ${HADOOP_HDFS_USER}
#create /mapred
su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /mapred" ${HADOOP_HDFS_USER}
su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 700 /mapred" ${HADOOP_HDFS_USER}
su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown ${HADOOP_MAPREDUCE_USER}:system /mapred" ${HADOOP_HDFS_USER}
if [ $? -eq 0 ]; then
echo "Completed."

73
hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-single-node.sh
View File

@ -17,16 +17,16 @@
# Script for setup HDFS file system for single node deployment
bin=`which $0`
bin=`dirname ${bin}`
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
export HADOOP_PREFIX=${bin}/..
if [ -e /etc/hadoop/hadoop-env.sh ]; then
. /etc/hadoop/hadoop-env.sh
if [ "$HADOOP_HOME" != "" ]; then
echo "Warning: \$HADOOP_HOME is deprecated."
echo
fi
. "$bin"/../libexec/hadoop-config.sh
usage() {
echo "
usage: $0 <parameters>
@ -38,7 +38,19 @@ usage: $0 <parameters>
exit 1
}
# Parse script parameters
template_generator() {
REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
cat $1 |
while read line ; do
while [[ "$line" =~ $REGEX ]] ; do
LHS=${BASH_REMATCH[1]}
RHS="$(eval echo "\"$LHS\"")"
line=${line//$LHS/$RHS}
done
echo $line >> $2
done
}
OPTS=$(getopt \
-n $0 \
-o '' \
@ -49,6 +61,10 @@ if [ $? != 0 ] ; then
usage
fi
if [ -e /etc/hadoop/hadoop-env.sh ]; then
. /etc/hadoop/hadoop-env.sh
fi
eval set -- "${OPTS}"
while true ; do
case "$1" in
@ -69,7 +85,6 @@ while true ; do
esac
done
# Interactive setup wizard
if [ "${AUTOMATED}" != "1" ]; then
echo "Welcome to Hadoop single node setup wizard"
echo
@ -119,68 +134,59 @@ SET_REBOOT=${SET_REBOOT:-y}
/etc/init.d/hadoop-jobtracker stop 2>/dev/null >/dev/null
/etc/init.d/hadoop-tasktracker stop 2>/dev/null >/dev/null
# Default settings
if [ "${SET_CONFIG}" == "y" ]; then
JAVA_HOME=${JAVA_HOME:-/usr/java/default}
HADOOP_NN_HOST=${HADOOP_NN_HOST:-hdfs://localhost:9000/}
HADOOP_NN_HOST=${HADOOP_NN_HOST:-localhost}
HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
HADOOP_JT_HOST=${HADOOP_JT_HOST:-localhost:9001}
HADOOP_JT_HOST=${HADOOP_JT_HOST:-localhost}
HADOOP_HDFS_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/hdfs}
HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/run/hadoop}
HADOOP_LOG_DIR="/var/log/hadoop"
HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
HADOOP_REPLICATION=${HADOOP_RELICATION:-1}
HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
# Setup config files
if [ "${SET_CONFIG}" == "y" ]; then
${HADOOP_PREFIX}/sbin/hadoop-setup-conf.sh --auto \
--hdfs-user=hdfs \
--mapreduce-user=mapred \
--conf-dir=${HADOOP_CONF_DIR} \
--datanode-dir=${HADOOP_DN_DIR} \
--hdfs-dir=${HADOOP_HDFS_DIR} \
--jobtracker-url=${HADOOP_JT_HOST} \
--jobtracker-host=${HADOOP_JT_HOST} \
--log-dir=${HADOOP_LOG_DIR} \
--pid-dir=${HADOOP_PID_DIR} \
--mapred-dir=${HADOOP_MAPRED_DIR} \
--namenode-dir=${HADOOP_NN_DIR} \
--namenode-url=${HADOOP_NN_HOST} \
--namenode-host=${HADOOP_NN_HOST} \
--replication=${HADOOP_REPLICATION}
fi
export HADOOP_CONF_DIR
# Format namenode
if [ ! -e ${HADOOP_NN_DIR} ]; then
rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null
mkdir -p ${HADOOP_HDFS_DIR}
chmod 755 ${HADOOP_HDFS_DIR}
chown hdfs:hadoop ${HADOOP_HDFS_DIR}
su -c '${HADOOP_PREFIX}/bin/hdfs --config ${HADOOP_CONF_DIR} namenode -format -clusterid hadoop' hdfs
/etc/init.d/hadoop-namenode format
elif [ "${SET_FORMAT}" == "y" ]; then
rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null
mkdir -p ${HADOOP_HDFS_DIR}
chmod 755 ${HADOOP_HDFS_DIR}
chown hdfs:hadoop ${HADOOP_HDFS_DIR}
rm -rf /var/lib/hadoop/hdfs/namenode
su -c '${HADOOP_PREFIX}/bin/hdfs --config ${HADOOP_CONF_DIR} namenode -format -clusterid hadoop' hdfs
rm -rf ${HADOOP_NN_DIR}
/etc/init.d/hadoop-namenode format
fi
# Start hdfs service
/etc/init.d/hadoop-namenode start
/etc/init.d/hadoop-datanode start
# Initialize file system structure
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /user/mapred' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /user/mapred' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /tmp' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 777 /tmp' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /jobtracker' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /jobtracker' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /user/mapred' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown mapred:mapred /user/mapred' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /tmp' hdfs
su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 777 /tmp' hdfs
# Start mapreduce service
/etc/init.d/hadoop-jobtracker start
/etc/init.d/hadoop-tasktracker start
# Toggle service startup on reboot
if [ "${SET_REBOOT}" == "y" ]; then
if [ -e /etc/debian_version ]; then
ln -sf ../init.d/hadoop-namenode /etc/rc2.d/S90hadoop-namenode
@ -203,7 +209,6 @@ if [ "${SET_REBOOT}" == "y" ]; then
fi
fi
# Shutdown service, if user choose to stop services after setup
if [ "${STARTUP}" != "y" ]; then
/etc/init.d/hadoop-namenode stop
/etc/init.d/hadoop-datanode stop

9
hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-datanode
View File

@ -27,10 +27,15 @@ source /etc/default/hadoop-env.sh
RETVAL=0
PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid"
desc="Hadoop datanode daemon"
HADOOP_PREFIX="/usr"
start() {
echo -n $"Starting $desc (hadoop-datanode): "
if [ -n "$HADOOP_SECURE_DN_USER" ]; then
daemon ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode
else
daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode
fi
RETVAL=$?
echo
[ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-datanode
@ -39,7 +44,11 @@ start() {
stop() {
echo -n $"Stopping $desc (hadoop-datanode): "
if [ -n "$HADOOP_SECURE_DN_USER" ]; then
daemon ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode
else
daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode
fi
RETVAL=$?
sleep 5
echo

1
hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-jobtracker
View File

@ -27,6 +27,7 @@ source /etc/default/hadoop-env.sh
RETVAL=0
PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid"
desc="Hadoop jobtracker daemon"
export HADOOP_PREFIX="/usr"
start() {
echo -n $"Starting $desc (hadoop-jobtracker): "

1
hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-namenode
View File

@ -27,6 +27,7 @@ source /etc/default/hadoop-env.sh
RETVAL=0
PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid"
desc="Hadoop namenode daemon"
export HADOOP_PREFIX="/usr"
start() {
echo -n $"Starting $desc (hadoop-namenode): "

1
hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-tasktracker
View File

@ -27,6 +27,7 @@ source /etc/default/hadoop-env.sh
RETVAL=0
PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid"
desc="Hadoop tasktracker daemon"
export HADOOP_PREFIX="/usr"
start() {
echo -n $"Starting $desc (hadoop-tasktracker): "

178
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/capacity-scheduler.xml Normal file
View File

@ -0,0 +1,178 @@
<?xml version="1.0"?>
<!-- This is the configuration file for the resource manager in Hadoop. -->
<!-- You can configure various scheduling parameters related to queues. -->
<!-- The properties for a queue follow a naming convention,such as, -->
<!-- mapred.capacity-scheduler.queue.<queue-name>.property-name. -->
<configuration>
<property>
<name>mapred.capacity-scheduler.maximum-system-jobs</name>
<value>3000</value>
<description>Maximum number of jobs in the system which can be initialized,
concurrently, by the CapacityScheduler.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.queue.default.capacity</name>
<value>100</value>
<description>Percentage of the number of slots in the cluster that are
to be available for jobs in this queue.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.queue.default.maximum-capacity</name>
<value>-1</value>
<description>
maximum-capacity defines a limit beyond which a queue cannot use the capacity of the cluster.
This provides a means to limit how much excess capacity a queue can use. By default, there is no limit.
The maximum-capacity of a queue can only be greater than or equal to its minimum capacity.
Default value of -1 implies a queue can use complete capacity of the cluster.
This property could be to curtail certain jobs which are long running in nature from occupying more than a
certain percentage of the cluster, which in the absence of pre-emption, could lead to capacity guarantees of
other queues being affected.
One important thing to note is that maximum-capacity is a percentage , so based on the cluster's capacity
the max capacity would change. So if large no of nodes or racks get added to the cluster , max Capacity in
absolute terms would increase accordingly.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.queue.default.supports-priority</name>
<value>false</value>
<description>If true, priorities of jobs will be taken into
account in scheduling decisions.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.queue.default.minimum-user-limit-percent</name>
<value>100</value>
<description> Each queue enforces a limit on the percentage of resources
allocated to a user at any given time, if there is competition for them.
This user limit can vary between a minimum and maximum value. The former
depends on the number of users who have submitted jobs, and the latter is
set to this property value. For example, suppose the value of this
property is 25. If two users have submitted jobs to a queue, no single
user can use more than 50% of the queue resources. If a third user submits
a job, no single user can use more than 33% of the queue resources. With 4
or more users, no user can use more than 25% of the queue's resources. A
value of 100 implies no user limits are imposed.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.queue.default.user-limit-factor</name>
<value>1</value>
<description>The multiple of the queue capacity which can be configured to
allow a single user to acquire more slots.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks</name>
<value>200000</value>
<description>The maximum number of tasks, across all jobs in the queue,
which can be initialized concurrently. Once the queue's jobs exceed this
limit they will be queued on disk.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks-per-user</name>
<value>100000</value>
<description>The maximum number of tasks per-user, across all the of the
user's jobs in the queue, which can be initialized concurrently. Once the
user's jobs exceed this limit they will be queued on disk.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.queue.default.init-accept-jobs-factor</name>
<value>10</value>
<description>The multipe of (maximum-system-jobs * queue-capacity) used to
determine the number of jobs which are accepted by the scheduler.
</description>
</property>
<!-- The default configuration settings for the capacity task scheduler -->
<!-- The default values would be applied to all the queues which don't have -->
<!-- the appropriate property for the particular queue -->
<property>
<name>mapred.capacity-scheduler.default-supports-priority</name>
<value>false</value>
<description>If true, priorities of jobs will be taken into
account in scheduling decisions by default in a job queue.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.default-minimum-user-limit-percent</name>
<value>100</value>
<description>The percentage of the resources limited to a particular user
for the job queue at any given point of time by default.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.default-user-limit-factor</name>
<value>1</value>
<description>The default multiple of queue-capacity which is used to
determine the amount of slots a single user can consume concurrently.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.default-maximum-active-tasks-per-queue</name>
<value>200000</value>
<description>The default maximum number of tasks, across all jobs in the
queue, which can be initialized concurrently. Once the queue's jobs exceed
this limit they will be queued on disk.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.default-maximum-active-tasks-per-user</name>
<value>100000</value>
<description>The default maximum number of tasks per-user, across all the of
the user's jobs in the queue, which can be initialized concurrently. Once
the user's jobs exceed this limit they will be queued on disk.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.default-init-accept-jobs-factor</name>
<value>10</value>
<description>The default multipe of (maximum-system-jobs * queue-capacity)
used to determine the number of jobs which are accepted by the scheduler.
</description>
</property>
<!-- Capacity scheduler Job Initialization configuration parameters -->
<property>
<name>mapred.capacity-scheduler.init-poll-interval</name>
<value>5000</value>
<description>The amount of time in miliseconds which is used to poll
the job queues for jobs to initialize.
</description>
</property>
<property>
<name>mapred.capacity-scheduler.init-worker-threads</name>
<value>5</value>
<description>Number of worker threads which would be used by
Initialization poller to initialize jobs in a set of queue.
If number mentioned in property is equal to number of job queues
then a single thread would initialize jobs in a queue. If lesser
then a thread would get a set of queues assigned. If the number
is greater then number of threads would be equal to number of
job queues.
</description>
</property>
</configuration>

7
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/commons-logging.properties Normal file
View File

@ -0,0 +1,7 @@
#Logging Implementation
#Log4J
org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger
#JDK Logger
#org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger

85
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/core-site.xml
View File

@ -1,27 +1,78 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>local.realm</name>
<value>${KERBEROS_REALM}</value>
</property>
<!-- file system properties -->
<property>
<name>fs.default.name</name>
<value>${HADOOP_NN_HOST}</value>
<value>hdfs://${HADOOP_NN_HOST}:8020</value>
<description>The name of the default file system. Either the
literal string "local" or a host:port for NDFS.
</description>
<final>true</final>
</property>
<property>
<name>fs.trash.interval</name>
<value>360</value>
<description>Number of minutes between trash checkpoints.
If zero, the trash feature is disabled.
</description>
</property>
<property>
<name>hadoop.security.auth_to_local</name>
<value>
RULE:[2:$1@$0]([jt]t@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
RULE:[2:$1@$0]([nd]n@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
RULE:[2:$1@$0](mapred@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
RULE:[2:$1@$0](hdfs@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
RULE:[2:$1@$0](mapredqa@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
RULE:[2:$1@$0](hdfsqa@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
DEFAULT
</value>
<description></description>
</property>
<property>
<name>hadoop.security.authentication</name>
<value>${SECURITY_TYPE}</value>
<description>
Set the authentication for the cluster. Valid values are: simple or
kerberos.
</description>
</property>
<property>
<name>hadoop.security.authorization</name>
<value>${SECURITY}</value>
<description>
Enable authorization for different protocols.
</description>
</property>
<property>
<name>hadoop.security.groups.cache.secs</name>
<value>14400</value>
</property>
<property>
<name>hadoop.kerberos.kinit.command</name>
<value>${KINIT}</value>
</property>
<property>
<name>hadoop.http.filter.initializers</name>
<value>org.apache.hadoop.http.lib.StaticUserWebFilter</value>
</property>
</configuration>

54
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh Normal file
View File

@ -0,0 +1,54 @@
# Set Hadoop-specific environment variables here.
# The only required environment variable is JAVA_HOME. All others are
# optional. When running a distributed configuration it is best to
# set JAVA_HOME in this file, so that it is correctly defined on
# remote nodes.
# The java implementation to use.
export JAVA_HOME=${JAVA_HOME}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
# Extra Java CLASSPATH elements. Automatically insert capacity-scheduler.
for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
if [ "$HADOOP_CLASSPATH" ]; then
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
else
export HADOOP_CLASSPATH=$f
fi
done
# The maximum amount of heap to use, in MB. Default is 1000.
#export HADOOP_HEAPSIZE=
#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
# Extra Java runtime options. Empty by default.
export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true $HADOOP_OPTS"
# Command specific options appended to HADOOP_OPTS when specified
export HADOOP_NAMENODE_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}"
HADOOP_JOBTRACKER_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dmapred.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
HADOOP_TASKTRACKER_OPTS="-Dsecurity.audit.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
HADOOP_DATANODE_OPTS="-Dsecurity.audit.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}"
export HADOOP_SECONDARYNAMENODE_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}"
# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
export HADOOP_CLIENT_OPTS="-Xmx128m ${HADOOP_CLIENT_OPTS}"
#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData ${HADOOP_JAVA_PLATFORM_OPTS}"
# On secure datanodes, user to run the datanode as after dropping privileges
export HADOOP_SECURE_DN_USER=${HADOOP_HDFS_USER}
# Where log files are stored. $HADOOP_HOME/logs by default.
export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
# Where log files are stored in the secure data environment.
export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
# The directory where pid files are stored. /tmp by default.
export HADOOP_PID_DIR=${HADOOP_PID_DIR}
export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
# A string representing this instance of hadoop. $USER by default.
export HADOOP_IDENT_STRING=$USER

118
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-policy.xml Normal file
View File

@ -0,0 +1,118 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>security.client.protocol.acl</name>
<value>*</value>
<description>ACL for ClientProtocol, which is used by user code
via the DistributedFileSystem.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.client.datanode.protocol.acl</name>
<value>*</value>
<description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
for block recovery.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.datanode.protocol.acl</name>
<value>*</value>
<description>ACL for DatanodeProtocol, which is used by datanodes to
communicate with the namenode.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.inter.datanode.protocol.acl</name>
<value>*</value>
<description>ACL for InterDatanodeProtocol, the inter-datanode protocol
for updating generation timestamp.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.namenode.protocol.acl</name>
<value>*</value>
<description>ACL for NamenodeProtocol, the protocol used by the secondary
namenode to communicate with the namenode.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.inter.tracker.protocol.acl</name>
<value>*</value>
<description>ACL for InterTrackerProtocol, used by the tasktrackers to
communicate with the jobtracker.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.job.submission.protocol.acl</name>
<value>*</value>
<description>ACL for JobSubmissionProtocol, used by job clients to
communciate with the jobtracker for job submission, querying job status etc.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.task.umbilical.protocol.acl</name>
<value>*</value>
<description>ACL for TaskUmbilicalProtocol, used by the map and reduce
tasks to communicate with the parent tasktracker.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.admin.operations.protocol.acl</name>
<value>${HADOOP_HDFS_USER}</value>
<description>ACL for AdminOperationsProtocol. Used for admin commands.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.refresh.usertogroups.mappings.protocol.acl</name>
<value>${HADOOP_HDFS_USER}</value>
<description>ACL for RefreshUserMappingsProtocol. Used to refresh
users mappings. The ACL is a comma-separated list of user and
group names. The user and group list is separated by a blank. For
e.g. "alice,bob users,wheel". A special value of "*" means all
users are allowed.</description>
</property>
<property>
<name>security.refresh.policy.protocol.acl</name>
<value>${HADOOP_HDFS_USER}</value>
<description>ACL for RefreshAuthorizationPolicyProtocol, used by the
dfsadmin and mradmin commands to refresh the security policy in-effect.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
</configuration>

225
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml Normal file
View File

@ -0,0 +1,225 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<!-- file system properties -->
<property>
<name>dfs.name.dir</name>
<value>${HADOOP_NN_DIR}</value>
<description>Determines where on the local filesystem the DFS name node
should store the name table. If this is a comma-delimited list
of directories then the name table is replicated in all of the
directories, for redundancy. </description>
<final>true</final>
</property>
<property>
<name>dfs.data.dir</name>
<value>${HADOOP_DN_DIR}</value>
<description>Determines where on the local filesystem an DFS data node
should store its blocks. If this is a comma-delimited
list of directories, then data will be stored in all named
directories, typically on different devices.
Directories that do not exist are ignored.
</description>
<final>true</final>
</property>
<property>
<name>dfs.safemode.threshold.pct</name>
<value>1.0f</value>
<description>
Specifies the percentage of blocks that should satisfy
the minimal replication requirement defined by dfs.replication.min.
Values less than or equal to 0 mean not to start in safe mode.
Values greater than 1 will make safe mode permanent.
</description>
</property>
<property>
<name>dfs.datanode.address</name>
<value>${HADOOP_DN_ADDR}</value>
</property>
<property>
<name>dfs.datanode.http.address</name>
<value>${HADOOP_DN_HTTP_ADDR}</value>
</property>
<property>
<name>dfs.http.address</name>
<value>${HADOOP_NN_HOST}:50070</value>
<description>The name of the default file system. Either the
literal string "local" or a host:port for NDFS.
</description>
<final>true</final>
</property>
<!-- Permissions configuration -->
<property>
<name>dfs.umaskmode</name>
<value>077</value>
<description>
The octal umask used when creating files and directories.
</description>
</property>
<property>
<name>dfs.block.access.token.enable</name>
<value>${SECURITY}</value>
<description>
Are access tokens are used as capabilities for accessing datanodes.
</description>
</property>
<property>
<name>dfs.namenode.kerberos.principal</name>
<value>nn/_HOST@${local.realm}</value>
<description>
Kerberos principal name for the NameNode
</description>
</property>
<property>
<name>dfs.secondary.namenode.kerberos.principal</name>
<value>nn/_HOST@${local.realm}</value>
<description>
Kerberos principal name for the secondary NameNode.
</description>
</property>
<property>
<name>dfs.namenode.kerberos.https.principal</name>
<value>host/_HOST@${local.realm}</value>
<description>
The Kerberos principal for the host that the NameNode runs on.
</description>
</property>
<property>
<name>dfs.secondary.namenode.kerberos.https.principal</name>
<value>host/_HOST@${local.realm}</value>
<description>
The Kerberos principal for the hostthat the secondary NameNode runs on.
</description>
</property>
<property>
<name>dfs.secondary.https.port</name>
<value>50490</value>
<description>The https port where secondary-namenode binds</description>
</property>
<property>
<name>dfs.datanode.kerberos.principal</name>
<value>dn/_HOST@${local.realm}</value>
<description>
The Kerberos principal that the DataNode runs as. "_HOST" is replaced by
the real host name.
</description>
</property>
<property>
<name>dfs.namenode.keytab.file</name>
<value>/etc/security/keytabs/nn.service.keytab</value>
<description>
Combined keytab file containing the namenode service and host principals.
</description>
</property>
<property>
<name>dfs.secondary.namenode.keytab.file</name>
<value>/etc/security/keytabs/nn.service.keytab</value>
<description>
Combined keytab file containing the namenode service and host principals.
</description>
</property>
<property>
<name>dfs.datanode.keytab.file</name>
<value>/etc/security/keytabs/dn.service.keytab</value>
<description>
The filename of the keytab file for the DataNode.
</description>
</property>
<property>
<name>dfs.https.port</name>
<value>50470</value>
<description>The https port where namenode binds</description>
</property>
<property>
<name>dfs.https.address</name>
<value>${HADOOP_NN_HOST}:50470</value>
<description>The https address where namenode binds</description>
</property>
<property>
<name>dfs.datanode.data.dir.perm</name>
<value>700</value>
<description>The permissions that should be there on dfs.data.dir
directories. The datanode will not come up if the permissions are
different on existing dfs.data.dir directories. If the directories
don't exist, they will be created with this permission.
</description>
</property>
<property>
<name>dfs.cluster.administrators</name>
<value>${HADOOP_HDFS_USER}</value>
<description>ACL for who all can view the default servlets in the HDFS</description>
</property>
<property>
<name>dfs.permissions.superusergroup</name>
<value>${HADOOP_GROUP}</value>
<description>The name of the group of super-users.</description>
</property>
<property>
<name>dfs.namenode.http-address</name>
<value>${HADOOP_NN_HOST}:50070</value>
<description>
The address and the base port where the dfs namenode web ui will listen on.
If the port is 0 then the server will start on a free port.
</description>
</property>
<property>
<name>dfs.namenode.https-address</name>
<value>${HADOOP_NN_HOST}:50470</value>
</property>
<property>
<name>dfs.secondary.http.address</name>
<value>${HADOOP_SNN_HOST}:50090</value>
<description>
The secondary namenode http server address and port.
If the port is 0 then the server will start on a free port.
</description>
</property>
<property>
<name>dfs.hosts</name>
<value>${HADOOP_CONF_DIR}/dfs.include</value>
<description>Names a file that contains a list of hosts that are
permitted to connect to the namenode. The full pathname of the file
must be specified. If the value is empty, all hosts are
permitted.</description>
</property>
<property>
<name>dfs.hosts.exclude</name>
<value>${HADOOP_CONF_DIR}/dfs.exclude</value>
<description>Names a file that contains a list of hosts that are
not permitted to connect to the namenode. The full pathname of the
file must be specified. If the value is empty, no hosts are
excluded.
</description>
</property>
</configuration>

12
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-queue-acls.xml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>mapred.queue.default.acl-submit-job</name>
<value>*</value>
</property>
<property>
<name>mapred.queue.default.acl-administer-jobs</name>
<value>*</value>
</property>
</configuration>

268
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml Normal file
View File

@ -0,0 +1,268 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>mapred.tasktracker.tasks.sleeptime-before-sigkill</name>
<value>250</value>
<description>Normally, this is the amount of time before killing
processes, and the recommended-default is 5.000 seconds - a value of
5000 here. In this case, we are using it solely to blast tasks before
killing them, and killing them very quickly (1/4 second) to guarantee
that we do not leave VMs around for later jobs.
</description>
</property>
<property>
<name>mapred.system.dir</name>
<value>/mapred/mapredsystem</value>
<final>true</final>
</property>
<property>
<name>mapred.job.tracker</name>
<value>${HADOOP_JT_HOST}:9000</value>
<final>true</final>
</property>
<property>
<name>mapred.job.tracker.http.address</name>
<value>${HADOOP_JT_HOST}:50030</value>
<final>true</final>
</property>
<property>
<name>mapred.local.dir</name>
<value>${HADOOP_MAPRED_DIR}</value>
<final>true</final>
</property>
<property>
<name>mapreduce.cluster.administrators</name>
<value>${HADOOP_MR_USER}</value>
</property>
<property>
<name>mapred.map.tasks.speculative.execution</name>
<value>false</value>
<description>If true, then multiple instances of some map tasks
may be executed in parallel.</description>
</property>
<property>
<name>mapred.reduce.tasks.speculative.execution</name>
<value>false</value>
<description>If true, then multiple instances of some reduce tasks
may be executed in parallel.</description>
</property>
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
<description>If the job outputs are to compressed as SequenceFiles, how
should they be compressed? Should be one of NONE, RECORD or BLOCK.
</description>
</property>
<property>
<name>jetty.connector</name>
<value>org.mortbay.jetty.nio.SelectChannelConnector</value>
</property>
<property>
<name>mapred.task.tracker.task-controller</name>
<value>${TASK_CONTROLLER}</value>
</property>
<property>
<name>mapred.child.root.logger</name>
<value>INFO,TLA</value>
</property>
<property>
<name>stream.tmpdir</name>
<value>${mapred.temp.dir}</value>
</property>
<property>
<name>mapred.child.java.opts</name>
<value>-server -Xmx640m -Djava.net.preferIPv4Stack=true</value>
</property>
<property>
<name>mapred.child.ulimit</name>
<value>8388608</value>
</property>
<property>
<name>mapred.job.tracker.persist.jobstatus.active</name>
<value>true</value>
<description>Indicates if persistency of job status information is
active or not.
</description>
</property>
<property>
<name>mapred.job.tracker.persist.jobstatus.dir</name>
<value>file:///${HADOOP_LOG_DIR}/${HADOOP_MR_USER}/jobstatus</value>
<description>The directory where the job status information is persisted
in a file system to be available after it drops of the memory queue and
between jobtracker restarts.
</description>
</property>
<property>
<name>mapred.job.tracker.history.completed.location</name>
<value>/mapred/history/done</value>
</property>
<property>
<name>mapred.heartbeats.in.second</name>
<value>200</value>
<description>to enable HADOOP:5784</description>
</property>
<property>
<name>mapreduce.tasktracker.outofband.heartbeat</name>
<value>true</value>
<description>to enable MAPREDUCE:270</description>
</property>
<property>
<name>mapred.jobtracker.maxtasks.per.job</name>
<value>200000</value>
<final>true</final>
<description>The maximum number of tasks for a single job.
A value of -1 indicates that there is no maximum.
</description>
</property>
<property>
<name>mapreduce.jobtracker.kerberos.principal</name>
<value>jt/_HOST@${local.realm}</value>
<description>
JT principal
</description>
</property>
<property>
<name>mapreduce.tasktracker.kerberos.principal</name>
<value>tt/_HOST@${local.realm}</value>
<description>
TT principal.
</description>
</property>
<property>
<name>hadoop.job.history.user.location</name>
<value>none</value>
</property>
<property>
<name>mapreduce.jobtracker.keytab.file</name>
<value>/etc/security/keytabs/jt.service.keytab</value>
<description>
The keytab for the jobtracker principal.
</description>
</property>
<property>
<name>mapreduce.tasktracker.keytab.file</name>
<value>/etc/security/keytabs/tt.service.keytab</value>
<description>The filename of the keytab for the task tracker</description>
</property>
<property>
<name>mapreduce.jobtracker.staging.root.dir</name>
<value>/user</value>
<description>The Path prefix for where the staging directories should be
placed. The next level is always the user's
name. It is a path in the default file system.
</description>
</property>
<property>
<name>mapreduce.job.acl-modify-job</name>
<value></value>
</property>
<property>
<name>mapreduce.job.acl-view-job</name>
<value>Dr.Who</value>
</property>
<property>
<name>mapreduce.tasktracker.group</name>
<value>${HADOOP_GROUP}</value>
<description>The group that the task controller uses for accessing the
task controller. The mapred user must be a member and users should *not*
be members.
</description>
</property>
<property>
<name>mapred.acls.enabled</name>
<value>true</value>
</property>
<property>
<name>mapred.jobtracker.taskScheduler</name>
<value>org.apache.hadoop.mapred.CapacityTaskScheduler</value>
</property>
<property>
<name>mapred.queue.names</name>
<value>default</value>
</property>
<!-- settings for the history server -->
<property>
<name>mapreduce.history.server.embedded</name>
<value>false</value>
</property>
<property>
<name>mapreduce.history.server.http.address</name>
<value>${HADOOP_JT_HOST}:51111</value>
</property>
<property>
<name>mapreduce.jobhistory.kerberos.principal</name>
<value>jt/_HOST@${local.realm}</value>
<description>history server principal</description>
</property>
<property>
<name>mapreduce.jobhistory.keytab.file</name>
<value>/etc/security/keytabs/jt.service.keytab</value>
<description>
The keytab for the jobtracker principal.
</description>
</property>
<property>
<name>mapred.hosts</name>
<value>${HADOOP_CONF_DIR}/mapred.include</value>
<description>Names a file that contains the list of nodes that may
connect to the jobtracker. If the value is empty, all hosts are
permitted.</description>
</property>
<property>
<name>mapred.hosts.exclude</name>
<value>${HADOOP_CONF_DIR}/mapred.exclude</value>
<description>Names a file that contains the list of hosts that
should be excluded by the jobtracker. If the value is empty, no
hosts are excluded.</description>
</property>
<property>
<name>mapred.jobtracker.retirejob.check</name>
<value>10000</value>
</property>
<property>
<name>mapred.jobtracker.retirejob.interval</name>
<value>0</value>
</property>
</configuration>

3
hadoop-common-project/hadoop-common/src/main/packages/templates/conf/taskcontroller.cfg Normal file
View File

@ -0,0 +1,3 @@
mapreduce.cluster.local.dir=${HADOOP_MAPRED_DIR}
mapreduce.tasktracker.group=${HADOOP_GROUP}
hadoop.log.dir=${HADOOP_LOG_DIR}/${HADOOP_MR_USER}

4
hadoop-common-project/hadoop-common/src/main/packages/update-hadoop-env.sh
View File

@ -134,7 +134,9 @@ if [ "${UNINSTALL}" -eq "1" ]; then
rm -rf ${HADOOP_PREFIX}/etc/hadoop
fi
rm -f /etc/default/hadoop-env.sh
if [ -d /etc/profile.d ]; then
rm -f /etc/profile.d/hadoop-env.sh
fi
else
# Create symlinks
if [ "${HADOOP_CONF_DIR}" != "${HADOOP_PREFIX}/etc/hadoop" ]; then
@ -142,7 +144,9 @@ else
ln -sf ${HADOOP_CONF_DIR} ${HADOOP_PREFIX}/etc/hadoop
fi
ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/default/hadoop-env.sh
if [ -d /etc/profile.d ]; then
ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/profile.d/hadoop-env.sh
fi
mkdir -p ${HADOOP_LOG_DIR}
chown root:hadoop ${HADOOP_LOG_DIR}

44
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.serializer;
import org.junit.Test;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
public class TestSerializationFactory {
@Test
public void testSerializerAvailability() {
Configuration conf = new Configuration();
SerializationFactory factory = new SerializationFactory(conf);
// Test that a valid serializer class is returned when its present
assertNotNull("A valid class must be returned for default Writable Serde",
factory.getSerializer(Writable.class));
assertNotNull("A valid class must be returned for default Writable serDe",
factory.getDeserializer(Writable.class));
// Test that a null is returned when none can be found.
assertNull("A null should be returned if there are no serializers found.",
factory.getSerializer(TestSerializationFactory.class));
assertNull("A null should be returned if there are no deserializers found",
factory.getDeserializer(TestSerializationFactory.class));
}
}

7
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -5,6 +5,11 @@ Trunk (unreleased changes)
HDFS-395. DFS Scalability: Incremental block reports. (Tomasz Nykiel
via hairong)
HDFS-2284. Add a new FileSystem, webhdfs://, for supporting write Http
access to HDFS. (szetszwo)
HDFS-2317. Support read access to HDFS in webhdfs. (szetszwo)
IMPROVEMENTS
HADOOP-7524 Change RPC to allow multiple protocols including multuple versions of the same protocol (sanjay Radia)
@ -1031,6 +1036,8 @@ Release 0.23.0 - Unreleased
HDFS-2289. Ensure jsvc is bundled with the HDFS distribution artifact.
(Alejandro Abdelnur via acmurthy)
HDFS-2323. start-dfs.sh script fails for tarball install (tomwhite)
BREAKDOWN OF HDFS-1073 SUBTASKS
HDFS-1521. Persist transaction ID on disk between NN restarts.

6
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
View File

@ -51,7 +51,7 @@ NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes)
echo "Starting namenodes on [$NAMENODES]"
"$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
--config "$HADOOP_CONF_DIR" \
--hostnames "$NAMENODES" \
--script "$bin/hdfs" start namenode $nameStartOpt
@ -64,7 +64,7 @@ if [ -n "$HADOOP_SECURE_DN_USER" ]; then
"Attempting to start secure cluster, skipping datanodes. " \
"Run start-secure-dns.sh as root to complete startup."
else
"$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
--config "$HADOOP_CONF_DIR" \
--script "$bin/hdfs" start datanode $dataStartOpt
fi
@ -84,7 +84,7 @@ if [ "$SECONDARY_NAMENODES" = '0.0.0.0' ] ; then
else
echo "Starting secondary namenodes [$SECONDARY_NAMENODES]"
"$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
--config "$HADOOP_CONF_DIR" \
--hostnames "$SECONDARY_NAMENODES" \
--script "$bin/hdfs" start secondarynamenode

2
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh
View File

@ -25,7 +25,7 @@ bin=`cd "$bin"; pwd`
. "$bin"/../libexec/hdfs-config.sh
if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt
"$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt
else
echo $usage
fi

6
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
View File

@ -27,7 +27,7 @@ NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes)
echo "Stopping namenodes on [$NAMENODES]"
"$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
--config "$HADOOP_CONF_DIR" \
--hostnames "$NAMENODES" \
--script "$bin/hdfs" stop namenode
@ -40,7 +40,7 @@ if [ -n "$HADOOP_SECURE_DN_USER" ]; then
"Attempting to stop secure cluster, skipping datanodes. " \
"Run stop-secure-dns.sh as root to complete shutdown."
else
"$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
--config "$HADOOP_CONF_DIR" \
--script "$bin/hdfs" stop datanode
fi
@ -60,7 +60,7 @@ if [ "$SECONDARY_NAMENODES" = '0.0.0.0' ] ; then
else
echo "Stopping secondary namenodes [$SECONDARY_NAMENODES]"
"$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
--config "$HADOOP_CONF_DIR" \
--hostnames "$SECONDARY_NAMENODES" \
--script "$bin/hdfs" stop secondarynamenode

2
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh
View File

@ -25,7 +25,7 @@ bin=`cd "$bin"; pwd`
. "$bin"/../libexec/hdfs-config.sh
if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode
"$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode
else
echo $usage
fi

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java
View File

@ -33,7 +33,7 @@
* is made on the successive read(). The normal input stream functions are
* connected to the currently active input stream.
*/
class ByteRangeInputStream extends FSInputStream {
public class ByteRangeInputStream extends FSInputStream {
/**
* This class wraps a URL to allow easy mocking when testing. The URL class
@ -71,7 +71,8 @@ enum StreamStatus {
StreamStatus status = StreamStatus.SEEK;
ByteRangeInputStream(final URL url) {
/** Create an input stream with the URL. */
public ByteRangeInputStream(final URL url) {
this(new URLOpener(url), new URLOpener(null));
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
View File

@ -802,7 +802,7 @@ private DFSOutputStream callAppend(HdfsFileStatus stat, String src,
*
* @see ClientProtocol#append(String, String)
*/
DFSOutputStream append(String src, int buffersize, Progressable progress)
public DFSOutputStream append(String src, int buffersize, Progressable progress)
throws IOException {
checkOpen();
HdfsFileStatus stat = getFileInfo(src);

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
View File

@ -98,7 +98,7 @@
* datanode from the original pipeline. The DataStreamer now
* starts sending packets from the dataQueue.
****************************************************************/
class DFSOutputStream extends FSOutputSummer implements Syncable {
public class DFSOutputStream extends FSOutputSummer implements Syncable {
private final DFSClient dfsClient;
private static final int MAX_PACKETS = 80; // each packet 64K, total 5MB
private Socket s;
@ -1707,7 +1707,7 @@ synchronized void setTestFilename(String newname) {
/**
* Returns the size of a file as it was when this stream was opened
*/
long getInitialLen() {
public long getInitialLen() {
return initialFileSize;
}

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -229,12 +229,11 @@ public boolean recoverLease(Path f) throws IOException {
return dfs.recoverLease(getPathName(f));
}
@SuppressWarnings("deprecation")
@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
statistics.incrementReadOps(1);
return new DFSClient.DFSDataInputStream(
dfs.open(getPathName(f), bufferSize, verifyChecksum, statistics));
dfs.open(getPathName(f), bufferSize, verifyChecksum));
}
/** This optional operation is not yet supported. */

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
View File

@ -250,7 +250,7 @@ public URI getUri() {
* @return namenode URL referring to the given path
* @throws IOException on error constructing the URL
*/
URL getNamenodeURL(String path, String query) throws IOException {
protected URL getNamenodeURL(String path, String query) throws IOException {
final URL url = new URL("http", nnAddr.getHostName(),
nnAddr.getPort(), path + '?' + query);
if (LOG.isTraceEnabled()) {
@ -317,6 +317,7 @@ protected String addDelegationTokenParam(String query) throws IOException {
@Override
public FSDataInputStream open(Path f, int buffersize) throws IOException {
f = f.makeQualified(getUri(), getWorkingDirectory());
String path = "/data" + ServletUtil.encodePath(f.toUri().getPath());
String query = addDelegationTokenParam("ugi=" + getEncodedUgiParameter());
URL u = getNamenodeURL(path, query);

55
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
View File

@ -18,7 +18,50 @@
package org.apache.hadoop.hdfs.server.datanode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SIMULATEDDATASTORAGE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SIMULATEDDATASTORAGE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STORAGEID_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SYNCONCLOSE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SYNCONCLOSE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_TRANSFERTO_ALLOWED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_TRANSFERTO_ALLOWED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_FEDERATION_NAMESERVICES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.server.common.Util.now;
import java.io.BufferedOutputStream;
@ -93,9 +136,11 @@
import org.apache.hadoop.hdfs.server.datanode.FSDataset.VolumeInfo;
import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
import org.apache.hadoop.hdfs.server.datanode.web.resources.DatanodeWebHdfsMethods;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.FileChecksumServlets;
import org.apache.hadoop.hdfs.server.namenode.StreamFile;
import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
@ -109,7 +154,8 @@
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtocolSignature;
@ -502,6 +548,11 @@ conf, new AccessControlList(conf.get(DFS_ADMIN, " ")),
this.infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);
this.infoServer.addServlet(null, "/blockScannerReport",
DataBlockScanner.Servlet.class);
infoServer.addJerseyResourcePackage(
DatanodeWebHdfsMethods.class.getPackage().getName()
+ ";" + Param.class.getPackage().getName(),
"/" + WebHdfsFileSystem.PATH_PREFIX + "/*");
this.infoServer.start();
}

228
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Normal file
View File

@ -0,0 +1,228 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.datanode.web.resources;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.EnumSet;
import javax.servlet.ServletContext;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.LengthParam;
import org.apache.hadoop.hdfs.web.resources.OffsetParam;
import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.hdfs.web.resources.PermissionParam;
import org.apache.hadoop.hdfs.web.resources.PostOpParam;
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
import org.apache.hadoop.io.IOUtils;
/** Web-hdfs DataNode implementation. */
@Path("")
public class DatanodeWebHdfsMethods {
public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class);
private @Context ServletContext context;
/** Handle HTTP PUT request. */
@PUT
@Path("{" + UriFsPathParam.NAME + ":.*}")
@Consumes({"*/*"})
@Produces({MediaType.APPLICATION_JSON})
public Response put(
final InputStream in,
@PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
@QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
final PutOpParam op,
@QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
final PermissionParam permission,
@QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
final OverwriteParam overwrite,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
final BufferSizeParam bufferSize,
@QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
final ReplicationParam replication,
@QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
final BlockSizeParam blockSize
) throws IOException, URISyntaxException {
if (LOG.isTraceEnabled()) {
LOG.trace(op + ": " + path
+ Param.toSortedString(", ", permission, overwrite, bufferSize,
replication, blockSize));
}
final String fullpath = path.getAbsolutePath();
final DataNode datanode = (DataNode)context.getAttribute("datanode");
switch(op.getValue()) {
case CREATE:
{
final Configuration conf = new Configuration(datanode.getConf());
final InetSocketAddress nnRpcAddr = NameNode.getAddress(conf);
final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
final FSDataOutputStream out = new FSDataOutputStream(dfsclient.create(
fullpath, permission.getFsPermission(),
overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
: EnumSet.of(CreateFlag.CREATE),
replication.getValue(), blockSize.getValue(), null,
bufferSize.getValue()), null);
try {
IOUtils.copyBytes(in, out, bufferSize.getValue());
} finally {
out.close();
}
final InetSocketAddress nnHttpAddr = NameNode.getHttpAddress(conf);
final URI uri = new URI(WebHdfsFileSystem.SCHEME, null,
nnHttpAddr.getHostName(), nnHttpAddr.getPort(), fullpath, null, null);
return Response.created(uri).type(MediaType.APPLICATION_JSON).build();
}
default:
throw new UnsupportedOperationException(op + " is not supported");
}
}
/** Handle HTTP POST request. */
@POST
@Path("{" + UriFsPathParam.NAME + ":.*}")
@Consumes({"*/*"})
@Produces({MediaType.APPLICATION_JSON})
public Response post(
final InputStream in,
@PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
@QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
final PostOpParam op,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
final BufferSizeParam bufferSize
) throws IOException, URISyntaxException {
if (LOG.isTraceEnabled()) {
LOG.trace(op + ": " + path
+ Param.toSortedString(", ", bufferSize));
}
final String fullpath = path.getAbsolutePath();
final DataNode datanode = (DataNode)context.getAttribute("datanode");
switch(op.getValue()) {
case APPEND:
{
final Configuration conf = new Configuration(datanode.getConf());
final InetSocketAddress nnRpcAddr = NameNode.getAddress(conf);
final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
final DFSOutputStream dfsout = dfsclient.append(fullpath,
bufferSize.getValue(), null);
final FSDataOutputStream out = new FSDataOutputStream(dfsout, null,
dfsout.getInitialLen());
try {
IOUtils.copyBytes(in, out, bufferSize.getValue());
} finally {
out.close();
}
return Response.ok().type(MediaType.APPLICATION_JSON).build();
}
default:
throw new UnsupportedOperationException(op + " is not supported");
}
}
/** Handle HTTP GET request. */
@GET
@Path("{" + UriFsPathParam.NAME + ":.*}")
@Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
public Response get(
@PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
@QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
final GetOpParam op,
@QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
final OffsetParam offset,
@QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
final LengthParam length,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
final BufferSizeParam bufferSize
) throws IOException, URISyntaxException {
if (LOG.isTraceEnabled()) {
LOG.trace(op + ": " + path
+ Param.toSortedString(", ", offset, length, bufferSize));
}
final String fullpath = path.getAbsolutePath();
final DataNode datanode = (DataNode)context.getAttribute("datanode");
switch(op.getValue()) {
case OPEN:
{
final Configuration conf = new Configuration(datanode.getConf());
final InetSocketAddress nnRpcAddr = NameNode.getAddress(conf);
final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
final DFSDataInputStream in = new DFSClient.DFSDataInputStream(
dfsclient.open(fullpath, bufferSize.getValue(), true));
in.seek(offset.getValue());
final StreamingOutput streaming = new StreamingOutput() {
@Override
public void write(final OutputStream out) throws IOException {
final Long n = length.getValue();
if (n == null) {
IOUtils.copyBytes(in, out, bufferSize.getValue());
} else {
IOUtils.copyBytes(in, out, n, false);
}
}
};
return Response.ok(streaming).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
default:
throw new UnsupportedOperationException(op + " is not supported");
}
}
}

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
View File

@ -335,6 +335,11 @@ protected void setRpcServerAddress(Configuration conf,
}
protected InetSocketAddress getHttpServerAddress(Configuration conf) {
return getHttpAddress(conf);
}
/** @return the NameNode HTTP address set in the conf. */
public static InetSocketAddress getHttpAddress(Configuration conf) {
return NetUtils.createSocketAddr(
conf.get(DFS_NAMENODE_HTTP_ADDRESS_KEY, DFS_NAMENODE_HTTP_ADDRESS_DEFAULT));
}

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
View File

@ -24,18 +24,20 @@
import javax.servlet.ServletContext;
import org.apache.commons.logging.Log;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.classification.InterfaceAudience;
/**
* Encapsulates the HTTP server started by the NameNode.
*/
@ -179,6 +181,11 @@ private static void setupServlets(HttpServer httpServer) {
FileChecksumServlets.RedirectServlet.class, false);
httpServer.addInternalServlet("contentSummary", "/contentSummary/*",
ContentSummaryServlet.class, false);
httpServer.addJerseyResourcePackage(
NamenodeWebHdfsMethods.class.getPackage().getName()
+ ";" + Param.class.getPackage().getName(),
"/" + WebHdfsFileSystem.PATH_PREFIX + "/*");
}
public static FSImage getFsImageFromContext(ServletContext context) {

400
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Normal file
View File

@ -0,0 +1,400 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode.web.resources;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.EnumSet;
import javax.servlet.ServletContext;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
import org.apache.hadoop.hdfs.web.resources.DstPathParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.GroupParam;
import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
import org.apache.hadoop.hdfs.web.resources.LengthParam;
import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
import org.apache.hadoop.hdfs.web.resources.OffsetParam;
import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
import org.apache.hadoop.hdfs.web.resources.OwnerParam;
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.hdfs.web.resources.PermissionParam;
import org.apache.hadoop.hdfs.web.resources.PostOpParam;
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
import org.apache.hadoop.net.NodeBase;
/** Web-hdfs NameNode implementation. */
@Path("")
public class NamenodeWebHdfsMethods {
private static final Log LOG = LogFactory.getLog(NamenodeWebHdfsMethods.class);
private @Context ServletContext context;
private static DatanodeInfo chooseDatanode(final NameNode namenode,
final String path, final HttpOpParam.Op op, final long openOffset
) throws IOException {
if (op == GetOpParam.Op.OPEN || op == PostOpParam.Op.APPEND) {
final NamenodeProtocols np = namenode.getRpcServer();
final HdfsFileStatus status = np.getFileInfo(path);
final long len = status.getLen();
if (op == GetOpParam.Op.OPEN && (openOffset < 0L || openOffset >= len)) {
throw new IOException("Offset=" + openOffset + " out of the range [0, "
+ len + "); " + op + ", path=" + path);
}
if (len > 0) {
final long offset = op == GetOpParam.Op.OPEN? openOffset: len - 1;
final LocatedBlocks locations = np.getBlockLocations(path, offset, 1);
final int count = locations.locatedBlockCount();
if (count > 0) {
return JspHelper.bestNode(locations.get(0));
}
}
}
return (DatanodeDescriptor)namenode.getNamesystem().getBlockManager(
).getDatanodeManager().getNetworkTopology().chooseRandom(
NodeBase.ROOT);
}
private static URI redirectURI(final NameNode namenode,
final String path, final HttpOpParam.Op op, final long openOffset,
final Param<?, ?>... parameters) throws URISyntaxException, IOException {
final DatanodeInfo dn = chooseDatanode(namenode, path, op, openOffset);
final String query = op.toQueryString() + Param.toSortedString("&", parameters);
final String uripath = "/" + WebHdfsFileSystem.PATH_PREFIX + path;
final URI uri = new URI("http", null, dn.getHostName(), dn.getInfoPort(),
uripath, query, null);
if (LOG.isTraceEnabled()) {
LOG.trace("redirectURI=" + uri);
}
return uri;
}
/** Handle HTTP PUT request. */
@PUT
@Path("{" + UriFsPathParam.NAME + ":.*}")
@Consumes({"*/*"})
@Produces({MediaType.APPLICATION_JSON})
public Response put(
final InputStream in,
@PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
@QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
final PutOpParam op,
@QueryParam(DstPathParam.NAME) @DefaultValue(DstPathParam.DEFAULT)
final DstPathParam dstPath,
@QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT)
final OwnerParam owner,
@QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT)
final GroupParam group,
@QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
final PermissionParam permission,
@QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
final OverwriteParam overwrite,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
final BufferSizeParam bufferSize,
@QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
final ReplicationParam replication,
@QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
final BlockSizeParam blockSize,
@QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
final ModificationTimeParam modificationTime,
@QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
final AccessTimeParam accessTime,
@QueryParam(RenameOptionSetParam.NAME) @DefaultValue(RenameOptionSetParam.DEFAULT)
final RenameOptionSetParam renameOptions
) throws IOException, URISyntaxException {
if (LOG.isTraceEnabled()) {
LOG.trace(op + ": " + path
+ Param.toSortedString(", ", dstPath, owner, group, permission,
overwrite, bufferSize, replication, blockSize,
modificationTime, accessTime, renameOptions));
}
final String fullpath = path.getAbsolutePath();
final NameNode namenode = (NameNode)context.getAttribute("name.node");
final NamenodeProtocols np = namenode.getRpcServer();
switch(op.getValue()) {
case CREATE:
{
final URI uri = redirectURI(namenode, fullpath, op.getValue(), -1L,
permission, overwrite, bufferSize, replication, blockSize);
return Response.temporaryRedirect(uri).build();
}
case MKDIRS:
{
final boolean b = np.mkdirs(fullpath, permission.getFsPermission(), true);
final String js = JsonUtil.toJsonString(PutOpParam.Op.MKDIRS, b);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
case RENAME:
{
final EnumSet<Options.Rename> s = renameOptions.getValue();
if (s.isEmpty()) {
@SuppressWarnings("deprecation")
final boolean b = np.rename(fullpath, dstPath.getValue());
final String js = JsonUtil.toJsonString(PutOpParam.Op.RENAME, b);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
} else {
np.rename(fullpath, dstPath.getValue(),
s.toArray(new Options.Rename[s.size()]));
return Response.ok().type(MediaType.APPLICATION_JSON).build();
}
}
case SETREPLICATION:
{
final boolean b = np.setReplication(fullpath, replication.getValue());
final String js = JsonUtil.toJsonString(PutOpParam.Op.SETREPLICATION, b);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
case SETOWNER:
{
np.setOwner(fullpath, owner.getValue(), group.getValue());
return Response.ok().type(MediaType.APPLICATION_JSON).build();
}
case SETPERMISSION:
{
np.setPermission(fullpath, permission.getFsPermission());
return Response.ok().type(MediaType.APPLICATION_JSON).build();
}
case SETTIMES:
{
np.setTimes(fullpath, modificationTime.getValue(), accessTime.getValue());
return Response.ok().type(MediaType.APPLICATION_JSON).build();
}
default:
throw new UnsupportedOperationException(op + " is not supported");
}
}
/** Handle HTTP POST request. */
@POST
@Path("{" + UriFsPathParam.NAME + ":.*}")
@Consumes({"*/*"})
@Produces({MediaType.APPLICATION_JSON})
public Response post(
final InputStream in,
@PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
@QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
final PostOpParam op,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
final BufferSizeParam bufferSize
) throws IOException, URISyntaxException {
if (LOG.isTraceEnabled()) {
LOG.trace(op + ": " + path
+ Param.toSortedString(", ", bufferSize));
}
final String fullpath = path.getAbsolutePath();
final NameNode namenode = (NameNode)context.getAttribute("name.node");
switch(op.getValue()) {
case APPEND:
{
final URI uri = redirectURI(namenode, fullpath, op.getValue(), -1L,
bufferSize);
return Response.temporaryRedirect(uri).build();
}
default:
throw new UnsupportedOperationException(op + " is not supported");
}
}
private static final UriFsPathParam ROOT = new UriFsPathParam("");
/** Handle HTTP GET request for the root. */
@GET
@Path("/")
@Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
public Response root(
@QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
final GetOpParam op,
@QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
final OffsetParam offset,
@QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
final LengthParam length,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
final BufferSizeParam bufferSize
) throws IOException, URISyntaxException {
return get(ROOT, op, offset, length, bufferSize);
}
/** Handle HTTP GET request. */
@GET
@Path("{" + UriFsPathParam.NAME + ":.*}")
@Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
public Response get(
@PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
@QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
final GetOpParam op,
@QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
final OffsetParam offset,
@QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
final LengthParam length,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
final BufferSizeParam bufferSize
) throws IOException, URISyntaxException {
if (LOG.isTraceEnabled()) {
LOG.trace(op + ", " + path
+ Param.toSortedString(", ", offset, length, bufferSize));
}
final NameNode namenode = (NameNode)context.getAttribute("name.node");
final String fullpath = path.getAbsolutePath();
final NamenodeProtocols np = namenode.getRpcServer();
switch(op.getValue()) {
case OPEN:
{
final URI uri = redirectURI(namenode, fullpath, op.getValue(),
offset.getValue(), offset, length, bufferSize);
return Response.temporaryRedirect(uri).build();
}
case GETFILESTATUS:
{
final HdfsFileStatus status = np.getFileInfo(fullpath);
final String js = JsonUtil.toJsonString(status);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
case LISTSTATUS:
{
final StreamingOutput streaming = getListingStream(np, fullpath);
return Response.ok(streaming).type(MediaType.APPLICATION_JSON).build();
}
default:
throw new UnsupportedOperationException(op + " is not supported");
}
}
private static DirectoryListing getDirectoryListing(final NamenodeProtocols np,
final String p, byte[] startAfter) throws IOException {
final DirectoryListing listing = np.getListing(p, startAfter, false);
if (listing == null) { // the directory does not exist
throw new FileNotFoundException("File " + p + " does not exist.");
}
return listing;
}
private static StreamingOutput getListingStream(final NamenodeProtocols np,
final String p) throws IOException {
final DirectoryListing first = getDirectoryListing(np, p,
HdfsFileStatus.EMPTY_NAME);
return new StreamingOutput() {
@Override
public void write(final OutputStream outstream) throws IOException {
final PrintStream out = new PrintStream(outstream);
out.print('[');
final HdfsFileStatus[] partial = first.getPartialListing();
if (partial.length > 0) {
out.print(JsonUtil.toJsonString(partial[0]));
}
for(int i = 1; i < partial.length; i++) {
out.println(',');
out.print(JsonUtil.toJsonString(partial[i]));
}
for(DirectoryListing curr = first; curr.hasMore(); ) {
curr = getDirectoryListing(np, p, curr.getLastName());
for(HdfsFileStatus s : curr.getPartialListing()) {
out.println(',');
out.print(JsonUtil.toJsonString(s));
}
}
out.println(']');
}
};
}
/** Handle HTTP DELETE request. */
@DELETE
@Path("{path:.*}")
@Produces(MediaType.APPLICATION_JSON)
public Response delete(
@PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
@QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT)
final DeleteOpParam op,
@QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT)
final RecursiveParam recursive
) throws IOException {
if (LOG.isTraceEnabled()) {
LOG.trace(op + ", " + path
+ Param.toSortedString(", ", recursive));
}
switch(op.getValue()) {
case DELETE:
final NameNode namenode = (NameNode)context.getAttribute("name.node");
final String fullpath = path.getAbsolutePath();
final boolean b = namenode.getRpcServer().delete(fullpath, recursive.getValue());
final String js = JsonUtil.toJsonString(DeleteOpParam.Op.DELETE, b);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
default:
throw new UnsupportedOperationException(op + " is not supported");
}
}
}

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
View File

@ -227,7 +227,10 @@ void printOut(String message) {
void printList(List<InetSocketAddress> list) {
StringBuilder buffer = new StringBuilder();
for (InetSocketAddress address : list) {
buffer.append(address.getHostName()).append(" ");
if (buffer.length() > 0) {
buffer.append(" ");
}
buffer.append(address.getHostName());
}
printOut(buffer.toString());
}

133
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java Normal file
View File

@ -0,0 +1,133 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web;
import java.util.Map;
import java.util.TreeMap;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.ipc.RemoteException;
import org.mortbay.util.ajax.JSON;
/** JSON Utilities */
public class JsonUtil {
private static final ThreadLocal<Map<String, Object>> jsonMap
= new ThreadLocal<Map<String, Object>>() {
@Override
protected Map<String, Object> initialValue() {
return new TreeMap<String, Object>();
}
@Override
public Map<String, Object> get() {
final Map<String, Object> m = super.get();
m.clear();
return m;
}
};
/** Convert an exception object to a Json string. */
public static String toJsonString(final Exception e) {
final Map<String, Object> m = jsonMap.get();
m.put("className", e.getClass().getName());
m.put("message", e.getMessage());
return JSON.toString(m);
}
/** Convert a Json map to a RemoteException. */
public static RemoteException toRemoteException(final Map<String, Object> m) {
final String className = (String)m.get("className");
final String message = (String)m.get("message");
return new RemoteException(className, message);
}
/** Convert a key-value pair to a Json string. */
public static String toJsonString(final Object key, final Object value) {
final Map<String, Object> m = jsonMap.get();
m.put(key instanceof String ? (String) key : key.toString(), value);
return JSON.toString(m);
}
/** Convert a FsPermission object to a string. */
public static String toString(final FsPermission permission) {
return String.format("%o", permission.toShort());
}
/** Convert a string to a FsPermission object. */
public static FsPermission toFsPermission(final String s) {
return new FsPermission(Short.parseShort(s, 8));
}
/** Convert a HdfsFileStatus object to a Json string. */
public static String toJsonString(final HdfsFileStatus status) {
final Map<String, Object> m = jsonMap.get();
if (status == null) {
m.put("isNull", true);
} else {
m.put("isNull", false);
m.put("localName", status.getLocalName());
m.put("isDir", status.isDir());
m.put("isSymlink", status.isSymlink());
if (status.isSymlink()) {
m.put("symlink", status.getSymlink());
}
m.put("len", status.getLen());
m.put("owner", status.getOwner());
m.put("group", status.getGroup());
m.put("permission", toString(status.getPermission()));
m.put("accessTime", status.getAccessTime());
m.put("modificationTime", status.getModificationTime());
m.put("blockSize", status.getBlockSize());
m.put("replication", status.getReplication());
}
return JSON.toString(m);
}
@SuppressWarnings("unchecked")
static Map<String, Object> parse(String jsonString) {
return (Map<String, Object>) JSON.parse(jsonString);
}
/** Convert a Json string to a HdfsFileStatus object. */
public static HdfsFileStatus toFileStatus(final Map<String, Object> m) {
if ((Boolean)m.get("isNull")) {
return null;
}
final String localName = (String) m.get("localName");
final boolean isDir = (Boolean) m.get("isDir");
final boolean isSymlink = (Boolean) m.get("isSymlink");
final byte[] symlink = isSymlink?
DFSUtil.string2Bytes((String)m.get("symlink")): null;
final long len = (Long) m.get("len");
final String owner = (String) m.get("owner");
final String group = (String) m.get("group");
final FsPermission permission = toFsPermission((String) m.get("permission"));
final long aTime = (Long) m.get("accessTime");
final long mTime = (Long) m.get("modificationTime");
final long blockSize = (Long) m.get("blockSize");
final short replication = (short) (long) (Long) m.get("replication");
return new HdfsFileStatus(len, isDir, replication, blockSize, mTime, aTime,
permission, owner, group,
symlink, DFSUtil.string2Bytes(localName));
}
}

386
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Normal file
View File

@ -0,0 +1,386 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web;
import java.io.BufferedOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.ByteRangeInputStream;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HftpFileSystem;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
import org.apache.hadoop.hdfs.web.resources.DstPathParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.GroupParam;
import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
import org.apache.hadoop.hdfs.web.resources.OwnerParam;
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.hdfs.web.resources.PermissionParam;
import org.apache.hadoop.hdfs.web.resources.PostOpParam;
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.mortbay.util.ajax.JSON;
/** A FileSystem for HDFS over the web. */
public class WebHdfsFileSystem extends HftpFileSystem {
/** File System URI: {SCHEME}://namenode:port/path/to/file */
public static final String SCHEME = "webhdfs";
/** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
public static final String PATH_PREFIX = SCHEME;
private UserGroupInformation ugi;
protected Path workingDir;
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf);
setConf(conf);
ugi = UserGroupInformation.getCurrentUser();
this.workingDir = getHomeDirectory();
}
@Override
public URI getUri() {
try {
return new URI(SCHEME, null, nnAddr.getHostName(), nnAddr.getPort(),
null, null, null);
} catch (URISyntaxException e) {
return null;
}
}
@Override
public Path getHomeDirectory() {
return makeQualified(new Path("/user/" + ugi.getShortUserName()));
}
@Override
public synchronized Path getWorkingDirectory() {
return workingDir;
}
@Override
public synchronized void setWorkingDirectory(final Path dir) {
String result = makeAbsolute(dir).toUri().getPath();
if (!DFSUtil.isValidName(result)) {
throw new IllegalArgumentException("Invalid DFS directory name " +
result);
}
workingDir = makeAbsolute(dir);
}
private Path makeAbsolute(Path f) {
return f.isAbsolute()? f: new Path(workingDir, f);
}
@SuppressWarnings("unchecked")
private static <T> T jsonParse(final InputStream in) throws IOException {
if (in == null) {
throw new IOException("The input stream is null.");
}
return (T)JSON.parse(new InputStreamReader(in));
}
private static void validateResponse(final HttpOpParam.Op op,
final HttpURLConnection conn) throws IOException {
final int code = conn.getResponseCode();
if (code != op.getExpectedHttpResponseCode()) {
final Map<String, Object> m;
try {
m = jsonParse(conn.getErrorStream());
} catch(IOException e) {
throw new IOException("Unexpected HTTP response: code=" + code + " != "
+ op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
+ ", message=" + conn.getResponseMessage(), e);
}
final RemoteException re = JsonUtil.toRemoteException(m);
throw re.unwrapRemoteException(AccessControlException.class,
DSQuotaExceededException.class,
FileAlreadyExistsException.class,
FileNotFoundException.class,
ParentNotDirectoryException.class,
SafeModeException.class,
NSQuotaExceededException.class,
UnresolvedPathException.class);
}
}
private URL toUrl(final HttpOpParam.Op op, final Path fspath,
final Param<?,?>... parameters) throws IOException {
//initialize URI path and query
final String path = "/" + PATH_PREFIX
+ makeQualified(fspath).toUri().getPath();
final String query = op.toQueryString()
+ Param.toSortedString("&", parameters);
final URL url = getNamenodeURL(path, query);
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
}
return url;
}
private HttpURLConnection httpConnect(final HttpOpParam.Op op, final Path fspath,
final Param<?,?>... parameters) throws IOException {
final URL url = toUrl(op, fspath, parameters);
//connect and get response
final HttpURLConnection conn = (HttpURLConnection)url.openConnection();
try {
conn.setRequestMethod(op.getType().toString());
conn.setDoOutput(op.getDoOutput());
if (op.getDoOutput()) {
conn.setRequestProperty("Expect", "100-Continue");
conn.setInstanceFollowRedirects(true);
}
conn.connect();
return conn;
} catch(IOException e) {
conn.disconnect();
throw e;
}
}
/**
* Run a http operation.
* Connect to the http server, validate response, and obtain the JSON output.
*
* @param op http operation
* @param fspath file system path
* @param parameters parameters for the operation
* @return a JSON object, e.g. Object[], Map<String, Object>, etc.
* @throws IOException
*/
private <T> T run(final HttpOpParam.Op op, final Path fspath,
final Param<?,?>... parameters) throws IOException {
final HttpURLConnection conn = httpConnect(op, fspath, parameters);
validateResponse(op, conn);
try {
return jsonParse(conn.getInputStream());
} finally {
conn.disconnect();
}
}
private FsPermission applyUMask(FsPermission permission) {
if (permission == null) {
permission = FsPermission.getDefault();
}
return permission.applyUMask(FsPermission.getUMask(getConf()));
}
private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
final Map<String, Object> json = run(op, f);
final HdfsFileStatus status = JsonUtil.toFileStatus(json);
if (status == null) {
throw new FileNotFoundException("File does not exist: " + f);
}
return status;
}
@Override
public FileStatus getFileStatus(Path f) throws IOException {
statistics.incrementReadOps(1);
return makeQualified(getHdfsFileStatus(f), f);
}
private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
f.getBlockSize(), f.getModificationTime(),
f.getAccessTime(),
f.getPermission(), f.getOwner(), f.getGroup(),
f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
}
@Override
public boolean mkdirs(Path f, FsPermission permission) throws IOException {
statistics.incrementWriteOps(1);
final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
final Map<String, Object> json = run(op, f,
new PermissionParam(applyUMask(permission)));
return (Boolean)json.get(op.toString());
}
@Override
public boolean rename(final Path src, final Path dst) throws IOException {
statistics.incrementWriteOps(1);
final HttpOpParam.Op op = PutOpParam.Op.RENAME;
final Map<String, Object> json = run(op, src,
new DstPathParam(makeQualified(dst).toUri().getPath()));
return (Boolean)json.get(op.toString());
}
@SuppressWarnings("deprecation")
@Override
public void rename(final Path src, final Path dst,
final Options.Rename... options) throws IOException {
statistics.incrementWriteOps(1);
final HttpOpParam.Op op = PutOpParam.Op.RENAME;
run(op, src, new DstPathParam(makeQualified(dst).toUri().getPath()),
new RenameOptionSetParam(options));
}
@Override
public void setOwner(final Path p, final String owner, final String group
) throws IOException {
if (owner == null && group == null) {
throw new IOException("owner == null && group == null");
}
statistics.incrementWriteOps(1);
final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
run(op, p, new OwnerParam(owner), new GroupParam(group));
}
@Override
public void setPermission(final Path p, final FsPermission permission
) throws IOException {
statistics.incrementWriteOps(1);
final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
run(op, p, new PermissionParam(permission));
}
@Override
public boolean setReplication(final Path p, final short replication
) throws IOException {
statistics.incrementWriteOps(1);
final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
final Map<String, Object> json = run(op, p,
new ReplicationParam(replication));
return (Boolean)json.get(op.toString());
}
@Override
public void setTimes(final Path p, final long mtime, final long atime
) throws IOException {
statistics.incrementWriteOps(1);
final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
run(op, p, new ModificationTimeParam(mtime), new AccessTimeParam(atime));
}
private FSDataOutputStream write(final HttpOpParam.Op op,
final HttpURLConnection conn, final int bufferSize) throws IOException {
return new FSDataOutputStream(new BufferedOutputStream(
conn.getOutputStream(), bufferSize), statistics) {
@Override
public void close() throws IOException {
try {
super.close();
} finally {
validateResponse(op, conn);
}
}
};
}
@Override
public FSDataOutputStream create(final Path f, final FsPermission permission,
final boolean overwrite, final int bufferSize, final short replication,
final long blockSize, final Progressable progress) throws IOException {
statistics.incrementWriteOps(1);
final HttpOpParam.Op op = PutOpParam.Op.CREATE;
final HttpURLConnection conn = httpConnect(op, f,
new PermissionParam(applyUMask(permission)),
new OverwriteParam(overwrite),
new BufferSizeParam(bufferSize),
new ReplicationParam(replication),
new BlockSizeParam(blockSize));
return write(op, conn, bufferSize);
}
@Override
public FSDataOutputStream append(final Path f, final int bufferSize,
final Progressable progress) throws IOException {
statistics.incrementWriteOps(1);
final HttpOpParam.Op op = PostOpParam.Op.APPEND;
final HttpURLConnection conn = httpConnect(op, f,
new BufferSizeParam(bufferSize));
return write(op, conn, bufferSize);
}
@Override
public boolean delete(Path f, boolean recursive) throws IOException {
final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
final Map<String, Object> json = run(op, f, new RecursiveParam(recursive));
return (Boolean)json.get(op.toString());
}
@Override
public FSDataInputStream open(final Path f, final int buffersize
) throws IOException {
statistics.incrementReadOps(1);
final HttpOpParam.Op op = GetOpParam.Op.OPEN;
final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
return new FSDataInputStream(new ByteRangeInputStream(url));
}
@Override
public FileStatus[] listStatus(final Path f) throws IOException {
statistics.incrementReadOps(1);
final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
final Object[] array = run(op, f);
//convert FileStatus
final FileStatus[] statuses = new FileStatus[array.length];
for(int i = 0; i < array.length; i++) {
@SuppressWarnings("unchecked")
final Map<String, Object> m = (Map<String, Object>)array[i];
statuses[i] = makeQualified(JsonUtil.toFileStatus(m), f);
}
return statuses;
}
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/AccessTimeParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Access time parameter. */
public class AccessTimeParam extends LongParam {
/** Parameter name. */
public static final String NAME = "accessTime";
/** Default parameter value. */
public static final String DEFAULT = "-1";
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public AccessTimeParam(final Long value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public AccessTimeParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/BlockSizeParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Block size parameter. */
public class BlockSizeParam extends LongParam {
/** Parameter name. */
public static final String NAME = "blockSize";
/** Default parameter value. */
public static final String DEFAULT = NULL;
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public BlockSizeParam(final Long value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public BlockSizeParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

51
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/BooleanParam.java Normal file
View File

@ -0,0 +1,51 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Boolean parameter. */
abstract class BooleanParam extends Param<Boolean, BooleanParam.Domain> {
static final String TRUE = "true";
static final String FALSE = "false";
BooleanParam(final Domain domain, final Boolean value) {
super(domain, value);
}
/** The domain of the parameter. */
static final class Domain extends Param.Domain<Boolean> {
Domain(final String paramName) {
super(paramName);
}
@Override
public String getDomain() {
return "<" + NULL + " | boolean>";
}
@Override
Boolean parse(final String str) {
if (TRUE.equalsIgnoreCase(str)) {
return true;
} else if (FALSE.equalsIgnoreCase(str)) {
return false;
}
throw new IllegalArgumentException("Failed to parse \"" + str
+ "\" to Boolean.");
}
}
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/BufferSizeParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Buffer size parameter. */
public class BufferSizeParam extends IntegerParam {
/** Parameter name. */
public static final String NAME = "bufferSize";
/** Default parameter value. */
public static final String DEFAULT = NULL;
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public BufferSizeParam(final Integer value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public BufferSizeParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

74
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java Normal file
View File

@ -0,0 +1,74 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.net.HttpURLConnection;
/** Http DELETE operation parameter. */
public class DeleteOpParam extends HttpOpParam<DeleteOpParam.Op> {
/** Parameter name. */
public static final String NAME = "deleteOp";
/** Delete operations. */
public static enum Op implements HttpOpParam.Op {
DELETE(HttpURLConnection.HTTP_OK),
NULL(HttpURLConnection.HTTP_NOT_IMPLEMENTED);
final int expectedHttpResponseCode;
Op(final int expectedHttpResponseCode) {
this.expectedHttpResponseCode = expectedHttpResponseCode;
}
@Override
public HttpOpParam.Type getType() {
return HttpOpParam.Type.DELETE;
}
@Override
public boolean getDoOutput() {
return false;
}
@Override
public int getExpectedHttpResponseCode() {
return expectedHttpResponseCode;
}
@Override
public String toQueryString() {
return NAME + "=" + this;
}
}
private static final Domain<Op> DOMAIN = new Domain<Op>(NAME, Op.class);
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public DeleteOpParam(final String str) {
super(DOMAIN, DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

43
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DstPathParam.java Normal file
View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import org.apache.hadoop.fs.Path;
/** Destination path parameter. */
public class DstPathParam extends StringParam {
/** Parameter name. */
public static final String NAME = "dstPath";
/** Default parameter value. */
public static final String DEFAULT = "";
private static final Domain DOMAIN = new Domain(NAME, null);
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public DstPathParam(final String str) {
super(DOMAIN, str == null || str.equals(DEFAULT)? null: new Path(str).toUri().getPath());
}
@Override
public String getName() {
return NAME;
}
}

46
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/EnumParam.java Normal file
View File

@ -0,0 +1,46 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.util.Arrays;
abstract class EnumParam<E extends Enum<E>> extends Param<E, EnumParam.Domain<E>> {
EnumParam(final Domain<E> domain, final E value) {
super(domain, value);
}
/** The domain of the parameter. */
static final class Domain<E extends Enum<E>> extends Param.Domain<E> {
private final Class<E> enumClass;
Domain(String name, final Class<E> enumClass) {
super(name);
this.enumClass = enumClass;
}
@Override
public final String getDomain() {
return Arrays.asList(enumClass.getEnumConstants()).toString();
}
@Override
final E parse(final String str) {
return Enum.valueOf(enumClass, str.toUpperCase());
}
}
}

86
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/EnumSetParam.java Normal file
View File

@ -0,0 +1,86 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.Iterator;
abstract class EnumSetParam<E extends Enum<E>> extends Param<EnumSet<E>, EnumSetParam.Domain<E>> {
/** Convert an EnumSet to a string of comma separated values. */
static <E extends Enum<E>> String toString(EnumSet<E> set) {
if (set == null || set.isEmpty()) {
return "";
} else {
final StringBuilder b = new StringBuilder();
final Iterator<E> i = set.iterator();
b.append(i.next());
for(; i.hasNext(); ) {
b.append(',').append(i.next());
}
return b.toString();
}
}
static <E extends Enum<E>> EnumSet<E> toEnumSet(final Class<E> clazz,
final E... values) {
final EnumSet<E> set = EnumSet.noneOf(clazz);
set.addAll(Arrays.asList(values));
return set;
}
EnumSetParam(final Domain<E> domain, final EnumSet<E> value) {
super(domain, value);
}
@Override
public String toString() {
return getName() + "=" + toString(value);
}
/** The domain of the parameter. */
static final class Domain<E extends Enum<E>> extends Param.Domain<EnumSet<E>> {
private final Class<E> enumClass;
Domain(String name, final Class<E> enumClass) {
super(name);
this.enumClass = enumClass;
}
@Override
public final String getDomain() {
return Arrays.asList(enumClass.getEnumConstants()).toString();
}
/** The string contains a comma separated values. */
@Override
final EnumSet<E> parse(final String str) {
final EnumSet<E> set = EnumSet.noneOf(enumClass);
if (!str.isEmpty()) {
for(int i, j = 0; j >= 0; ) {
i = j;
j = str.indexOf(',', i+1);
final String sub = j >= 0? str.substring(i, j): str.substring(i);
set.add(Enum.valueOf(enumClass, sub.trim().toUpperCase()));
}
}
return set;
}
}
}

59
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java Normal file
View File

@ -0,0 +1,59 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.io.FileNotFoundException;
import java.io.IOException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
import javax.ws.rs.ext.Provider;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.web.JsonUtil;
/** Handle exceptions. */
@Provider
public class ExceptionHandler implements ExceptionMapper<Exception> {
public static final Log LOG = LogFactory.getLog(ExceptionHandler.class);
@Override
public Response toResponse(final Exception e) {
if (LOG.isTraceEnabled()) {
LOG.trace("GOT EXCEPITION", e);
}
final Response.Status s;
if (e instanceof SecurityException) {
s = Response.Status.UNAUTHORIZED;
} else if (e instanceof FileNotFoundException) {
s = Response.Status.NOT_FOUND;
} else if (e instanceof IOException) {
s = Response.Status.FORBIDDEN;
} else if (e instanceof UnsupportedOperationException) {
s = Response.Status.BAD_REQUEST;
} else {
s = Response.Status.INTERNAL_SERVER_ERROR;
}
final String js = JsonUtil.toJsonString(e);
return Response.status(s).type(MediaType.APPLICATION_JSON).entity(js).build();
}
}

77
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java Normal file
View File

@ -0,0 +1,77 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.net.HttpURLConnection;
/** Http GET operation parameter. */
public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
/** Parameter name. */
public static final String NAME = "getOp";
/** Get operations. */
public static enum Op implements HttpOpParam.Op {
OPEN(HttpURLConnection.HTTP_OK),
GETFILESTATUS(HttpURLConnection.HTTP_OK),
LISTSTATUS(HttpURLConnection.HTTP_OK),
NULL(HttpURLConnection.HTTP_NOT_IMPLEMENTED);
final int expectedHttpResponseCode;
Op(final int expectedHttpResponseCode) {
this.expectedHttpResponseCode = expectedHttpResponseCode;
}
@Override
public HttpOpParam.Type getType() {
return HttpOpParam.Type.GET;
}
@Override
public boolean getDoOutput() {
return false;
}
@Override
public int getExpectedHttpResponseCode() {
return expectedHttpResponseCode;
}
@Override
public String toQueryString() {
return NAME + "=" + this;
}
}
private static final Domain<Op> DOMAIN = new Domain<Op>(NAME, Op.class);
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public GetOpParam(final String str) {
super(DOMAIN, DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

41
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GroupParam.java Normal file
View File

@ -0,0 +1,41 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Group parameter. */
public class GroupParam extends StringParam {
/** Parameter name. */
public static final String NAME = "group";
/** Default parameter value. */
public static final String DEFAULT = "";
private static final Domain DOMAIN = new Domain(NAME, null);
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public GroupParam(final String str) {
super(DOMAIN, str == null || str.equals(DEFAULT)? null: str);
}
@Override
public String getName() {
return NAME;
}
}

52
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java Normal file
View File

@ -0,0 +1,52 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Http operation parameter. */
public abstract class HttpOpParam<E extends Enum<E> & HttpOpParam.Op> extends EnumParam<E> {
/** Default parameter value. */
public static final String DEFAULT = NULL;
/** Http operation types */
public static enum Type {
GET, PUT, POST, DELETE;
}
/** Http operation interface. */
public static interface Op {
/** @return the Http operation type. */
public Type getType();
/** @return true if the operation has output. */
public boolean getDoOutput();
/** @return true if the operation has output. */
public int getExpectedHttpResponseCode();
/** @return a URI query string. */
public String toQueryString();
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
HttpOpParam(final Domain<E> domain, final E value) {
super(domain, value);
}
}

60
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/IntegerParam.java Normal file
View File

@ -0,0 +1,60 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Integer parameter. */
abstract class IntegerParam extends Param<Integer, IntegerParam.Domain> {
IntegerParam(final Domain domain, final Integer value) {
super(domain, value);
}
@Override
public String toString() {
return getName() + "=" + domain.toString(getValue());
}
/** The domain of the parameter. */
static final class Domain extends Param.Domain<Integer> {
/** The radix of the number. */
final int radix;
Domain(final String paramName) {
this(paramName, 10);
}
Domain(final String paramName, final int radix) {
super(paramName);
this.radix = radix;
}
@Override
public String getDomain() {
return "<" + NULL + " | int in radix " + radix + ">";
}
@Override
Integer parse(final String str) {
return NULL.equals(str)? null: Integer.parseInt(str, radix);
}
/** Convert an Integer to a String. */
String toString(final Integer n) {
return n == null? NULL: Integer.toString(n, radix);
}
}
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/LengthParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Length parameter. */
public class LengthParam extends LongParam {
/** Parameter name. */
public static final String NAME = "length";
/** Default parameter value. */
public static final String DEFAULT = NULL;
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public LengthParam(final Long value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public LengthParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

60
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/LongParam.java Normal file
View File

@ -0,0 +1,60 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Long parameter. */
abstract class LongParam extends Param<Long, LongParam.Domain> {
LongParam(final Domain domain, final Long value) {
super(domain, value);
}
@Override
public String toString() {
return getName() + "=" + domain.toString(getValue());
}
/** The domain of the parameter. */
static final class Domain extends Param.Domain<Long> {
/** The radix of the number. */
final int radix;
Domain(final String paramName) {
this(paramName, 10);
}
Domain(final String paramName, final int radix) {
super(paramName);
this.radix = radix;
}
@Override
public String getDomain() {
return "<" + NULL + " | short in radix " + radix + ">";
}
@Override
Long parse(final String str) {
return NULL.equals(str)? null: Long.parseLong(str, radix);
}
/** Convert a Short to a String. */
String toString(final Long n) {
return n == null? NULL: Long.toString(n, radix);
}
}
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ModificationTimeParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Modification time parameter. */
public class ModificationTimeParam extends LongParam {
/** Parameter name. */
public static final String NAME = "modificationTime";
/** Default parameter value. */
public static final String DEFAULT = "-1";
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public ModificationTimeParam(final Long value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public ModificationTimeParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/OffsetParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Offset parameter. */
public class OffsetParam extends LongParam {
/** Parameter name. */
public static final String NAME = "offset";
/** Default parameter value. */
public static final String DEFAULT = "0";
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public OffsetParam(final Long value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public OffsetParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Recursive parameter. */
public class OverwriteParam extends BooleanParam {
/** Parameter name. */
public static final String NAME = "overwrite";
/** Default parameter value. */
public static final String DEFAULT = FALSE;
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public OverwriteParam(final Boolean value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public OverwriteParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

41
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/OwnerParam.java Normal file
View File

@ -0,0 +1,41 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Owner parameter. */
public class OwnerParam extends StringParam {
/** Parameter name. */
public static final String NAME = "owner";
/** Default parameter value. */
public static final String DEFAULT = "";
private static final Domain DOMAIN = new Domain(NAME, null);
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public OwnerParam(final String str) {
super(DOMAIN, str == null || str.equals(DEFAULT)? null: str);
}
@Override
public String getName() {
return NAME;
}
}

104
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/Param.java Normal file
View File

@ -0,0 +1,104 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.util.Arrays;
import java.util.Comparator;
/** Base class of parameters. */
public abstract class Param<T, D extends Param.Domain<T>> {
static final String NULL = "null";
static final Comparator<Param<?,?>> NAME_CMP = new Comparator<Param<?,?>>() {
@Override
public int compare(Param<?, ?> left, Param<?, ?> right) {
return left.getName().compareTo(right.getName());
}
};
/** Convert the parameters to a sorted String. */
public static String toSortedString(final String separator,
final Param<?, ?>... parameters) {
Arrays.sort(parameters, NAME_CMP);
final StringBuilder b = new StringBuilder();
for(Param<?, ?> p : parameters) {
if (p.getValue() != null) {
b.append(separator).append(p);
}
}
return b.toString();
}
/** The domain of the parameter. */
final D domain;
/** The actual parameter value. */
final T value;
Param(final D domain, final T value) {
this.domain = domain;
this.value = value;
}
/** @return the parameter value. */
public final T getValue() {
return value;
}
/** @return the parameter name. */
public abstract String getName();
@Override
public String toString() {
return getName() + "=" + value;
}
/** Base class of parameter domains. */
static abstract class Domain<T> {
/** Parameter name. */
final String paramName;
Domain(final String paramName) {
this.paramName = paramName;
}
/** @return the parameter name. */
public final String getParamName() {
return paramName;
}
/** @return a string description of the domain of the parameter. */
public abstract String getDomain();
/** @return the parameter value represented by the string. */
abstract T parse(String str);
/** Parse the given string.
* @return the parameter value represented by the string.
*/
public final T parse(final String varName, final String str) {
try {
return str != null && str.trim().length() > 0 ? parse(str) : null;
} catch(Exception e) {
throw new IllegalArgumentException("Failed to parse \"" + str
+ "\" for the parameter " + varName
+ ". The value must be in the domain " + getDomain(), e);
}
}
}
}

57
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java Normal file
View File

@ -0,0 +1,57 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import org.apache.hadoop.fs.permission.FsPermission;
/** Permission parameter, use a Short to represent a FsPermission. */
public class PermissionParam extends ShortParam {
/** Parameter name. */
public static final String NAME = "permission";
/** Default parameter value. */
public static final String DEFAULT = NULL;
private static final Domain DOMAIN = new Domain(NAME, 8);
/**
* Constructor.
* @param value the parameter value.
*/
public PermissionParam(final FsPermission value) {
super(DOMAIN, value == null? null: value.toShort());
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public PermissionParam(final String str) {
super(DOMAIN, DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
/** @return the represented FsPermission. */
public FsPermission getFsPermission() {
final Short mode = getValue();
return mode == null? FsPermission.getDefault(): new FsPermission(mode);
}
}

74
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java Normal file
View File

@ -0,0 +1,74 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.net.HttpURLConnection;
/** Http POST operation parameter. */
public class PostOpParam extends HttpOpParam<PostOpParam.Op> {
/** Parameter name. */
public static final String NAME = "postOp";
/** Post operations. */
public static enum Op implements HttpOpParam.Op {
APPEND(HttpURLConnection.HTTP_OK),
NULL(HttpURLConnection.HTTP_NOT_IMPLEMENTED);
final int expectedHttpResponseCode;
Op(final int expectedHttpResponseCode) {
this.expectedHttpResponseCode = expectedHttpResponseCode;
}
@Override
public Type getType() {
return Type.POST;
}
@Override
public boolean getDoOutput() {
return true;
}
@Override
public int getExpectedHttpResponseCode() {
return expectedHttpResponseCode;
}
/** @return a URI query string. */
public String toQueryString() {
return NAME + "=" + this;
}
}
private static final Domain<Op> DOMAIN = new Domain<PostOpParam.Op>(NAME, Op.class);
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public PostOpParam(final String str) {
super(DOMAIN, DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

84
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java Normal file
View File

@ -0,0 +1,84 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.net.HttpURLConnection;
/** Http POST operation parameter. */
public class PutOpParam extends HttpOpParam<PutOpParam.Op> {
/** Parameter name. */
public static final String NAME = "putOp";
/** Put operations. */
public static enum Op implements HttpOpParam.Op {
CREATE(true, HttpURLConnection.HTTP_CREATED),
MKDIRS(false, HttpURLConnection.HTTP_OK),
RENAME(false, HttpURLConnection.HTTP_OK),
SETREPLICATION(false, HttpURLConnection.HTTP_OK),
SETOWNER(false, HttpURLConnection.HTTP_OK),
SETPERMISSION(false, HttpURLConnection.HTTP_OK),
SETTIMES(false, HttpURLConnection.HTTP_OK),
NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED);
final boolean doOutput;
final int expectedHttpResponseCode;
Op(final boolean doOutput, final int expectedHttpResponseCode) {
this.doOutput = doOutput;
this.expectedHttpResponseCode = expectedHttpResponseCode;
}
@Override
public HttpOpParam.Type getType() {
return HttpOpParam.Type.PUT;
}
@Override
public boolean getDoOutput() {
return doOutput;
}
@Override
public int getExpectedHttpResponseCode() {
return expectedHttpResponseCode;
}
@Override
public String toQueryString() {
return NAME + "=" + this;
}
}
private static final Domain<Op> DOMAIN = new Domain<Op>(NAME, Op.class);
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public PutOpParam(final String str) {
super(DOMAIN, DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/RecursiveParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Recursive parameter. */
public class RecursiveParam extends BooleanParam {
/** Parameter name. */
public static final String NAME = "recursive";
/** Default parameter value. */
public static final String DEFAULT = FALSE;
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public RecursiveParam(final Boolean value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public RecursiveParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

52
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/RenameOptionSetParam.java Normal file
View File

@ -0,0 +1,52 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import org.apache.hadoop.fs.Options;
/** Rename option set parameter. */
public class RenameOptionSetParam extends EnumSetParam<Options.Rename> {
/** Parameter name. */
public static final String NAME = "renameOptions";
/** Default parameter value. */
public static final String DEFAULT = "";
private static final Domain<Options.Rename> DOMAIN = new Domain<Options.Rename>(
NAME, Options.Rename.class);
/**
* Constructor.
* @param options rename options.
*/
public RenameOptionSetParam(final Options.Rename... options) {
super(DOMAIN, toEnumSet(Options.Rename.class, options));
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public RenameOptionSetParam(final String str) {
super(DOMAIN, DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ReplicationParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Replication parameter. */
public class ReplicationParam extends ShortParam {
/** Parameter name. */
public static final String NAME = "replication";
/** Default parameter value. */
public static final String DEFAULT = NULL;
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public ReplicationParam(final Short value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public ReplicationParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

60
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ShortParam.java Normal file
View File

@ -0,0 +1,60 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** Short parameter. */
abstract class ShortParam extends Param<Short, ShortParam.Domain> {
ShortParam(final Domain domain, final Short value) {
super(domain, value);
}
@Override
public String toString() {
return getName() + "=" + domain.toString(getValue());
}
/** The domain of the parameter. */
static final class Domain extends Param.Domain<Short> {
/** The radix of the number. */
final int radix;
Domain(final String paramName) {
this(paramName, 10);
}
Domain(final String paramName, final int radix) {
super(paramName);
this.radix = radix;
}
@Override
public String getDomain() {
return "<" + NULL + " | short in radix " + radix + ">";
}
@Override
Short parse(final String str) {
return NULL.equals(str)? null: Short.parseShort(str, radix);
}
/** Convert a Short to a String. */
String toString(final Short n) {
return n == null? NULL: Integer.toString(n, radix);
}
}
}

54
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/StringParam.java Normal file
View File

@ -0,0 +1,54 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.util.regex.Pattern;
/** String parameter. */
abstract class StringParam extends Param<String, StringParam.Domain> {
StringParam(final Domain domain, String str) {
super(domain, domain.parse(str));
}
/** The domain of the parameter. */
static final class Domain extends Param.Domain<String> {
/** The pattern defining the domain; null . */
private final Pattern pattern;
Domain(final String paramName, final Pattern pattern) {
super(paramName);
this.pattern = pattern;
}
@Override
public final String getDomain() {
return pattern == null ? "<String>" : pattern.pattern();
}
@Override
final String parse(final String str) {
if (pattern != null) {
if (!pattern.matcher(str).matches()) {
throw new IllegalArgumentException("Invalid value: \"" + str
+ "\" does not belong to the domain " + getDomain());
}
}
return str;
}
}
}

45
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UriFsPathParam.java Normal file
View File

@ -0,0 +1,45 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** The FileSystem path parameter. */
public class UriFsPathParam extends StringParam {
/** Parameter name. */
public static final String NAME = "path";
private static final Domain DOMAIN = new Domain(NAME, null);
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public UriFsPathParam(String str) {
super(DOMAIN, str);
}
@Override
public String getName() {
return NAME;
}
/** @return the absolute path. */
public final String getAbsolutePath() {
final String path = getValue(); //The first / has been stripped out.
return path == null? null: "/" + path;
}
}

41
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java Normal file
View File

@ -0,0 +1,41 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** User parameter. */
public class UserParam extends StringParam {
/** Parameter name. */
public static final String NAME = "user.name";
/** Default parameter value. */
public static final String DEFAULT = "";
private static final Domain DOMAIN = new Domain(NAME, null);
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public UserParam(final String str) {
super(DOMAIN, str == null || str.equals(DEFAULT)? null: str);
}
@Override
public String getName() {
return NAME;
}
}

73
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserProvider.java Normal file
View File

@ -0,0 +1,73 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
import java.lang.reflect.Type;
import java.security.Principal;
import javax.ws.rs.core.Context;
import javax.ws.rs.ext.Provider;
import com.sun.jersey.api.core.HttpContext;
import com.sun.jersey.core.spi.component.ComponentContext;
import com.sun.jersey.core.spi.component.ComponentScope;
import com.sun.jersey.server.impl.inject.AbstractHttpContextInjectable;
import com.sun.jersey.spi.inject.Injectable;
import com.sun.jersey.spi.inject.InjectableProvider;
@Provider
public class UserProvider extends AbstractHttpContextInjectable<Principal>
implements InjectableProvider<Context, Type> {
@Override
public Principal getValue(final HttpContext context) {
//get principal from the request
final Principal principal = context.getRequest().getUserPrincipal();
if (principal != null) {
return principal;
}
//get username from the parameter
final String username = context.getRequest().getQueryParameters().getFirst(
UserParam.NAME);
if (username != null) {
final UserParam userparam = new UserParam(username);
return new Principal() {
@Override
public String getName() {
return userparam.getValue();
}
};
}
//user not found
return null;
}
@Override
public ComponentScope getScope() {
return ComponentScope.PerRequest;
}
@Override
public Injectable<Principal> getInjectable(
final ComponentContext componentContext, final Context context,
final Type type) {
return type.equals(Principal.class)? this : null;
}
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -815,6 +815,8 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
long[] simulatedCapacities,
boolean setupHostsFile,
boolean checkDataNodeAddrConfig) throws IOException {
conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1");
int curDatanodesNum = dataNodes.size();
// for mincluster's the default initialDelay for BRs is 0
if (conf.get(DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY) == null) {

79
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java Normal file
View File

@ -0,0 +1,79 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web;
import java.net.URI;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSMainOperationsBaseTest;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.web.resources.DatanodeWebHdfsMethods;
import org.apache.hadoop.hdfs.web.resources.ExceptionHandler;
import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
public class TestFSMainOperationsWebHdfs extends FSMainOperationsBaseTest {
{
((Log4JLogger)ExceptionHandler.LOG).getLogger().setLevel(Level.ALL);
((Log4JLogger)DatanodeWebHdfsMethods.LOG).getLogger().setLevel(Level.ALL);
}
private static MiniDFSCluster cluster = null;
private static Path defaultWorkingDirectory;
@BeforeClass
public static void setupCluster() {
Configuration conf = new Configuration();
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
cluster.waitActive();
final String uri = WebHdfsFileSystem.SCHEME + "://"
+ conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
fSys = FileSystem.get(new URI(uri), conf);
defaultWorkingDirectory = fSys.getWorkingDirectory();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@AfterClass
public static void shutdownCluster() {
if (cluster != null) {
cluster.shutdown();
cluster = null;
}
}
@Override
protected Path getDefaultWorkingDirectory() {
return defaultWorkingDirectory;
}
//The following test failed since WebHdfsFileSystem did not support
//authentication.
//Disable it.
@Test
public void testListStatusThrowsExceptionForUnreadableDir() {}
}

55
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java Normal file
View File

@ -0,0 +1,55 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.junit.Assert;
import org.junit.Test;
public class TestJsonUtil {
static FileStatus toFileStatus(HdfsFileStatus f, String parent) {
return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
f.getPermission(), f.getOwner(), f.getGroup(),
new Path(f.getFullName(parent)));
}
@Test
public void testHdfsFileStatus() {
final long now = System.currentTimeMillis();
final String parent = "/dir";
final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L<<26,
now, now + 10, new FsPermission((short)0644), "user", "group",
DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo"));
final FileStatus fstatus = toFileStatus(status, parent);
System.out.println("status = " + status);
System.out.println("fstatus = " + fstatus);
final String json = JsonUtil.toJsonString(status);
System.out.println("json = " + json.replace(",", ",\n "));
final HdfsFileStatus s2 = JsonUtil.toFileStatus(JsonUtil.parse(json));
final FileStatus fs2 = toFileStatus(s2, parent);
System.out.println("s2 = " + s2);
System.out.println("fs2 = " + fs2);
Assert.assertEquals(fstatus, fs2);
}
}

56
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java Normal file
View File

@ -0,0 +1,56 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
public class TestWebHdfsFileSystemContract extends FileSystemContractBaseTest {
private static final Configuration conf = new Configuration();
private static final MiniDFSCluster cluster;
private String defaultWorkingDirectory;
static {
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
cluster.waitActive();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
protected void setUp() throws Exception {
final String uri = WebHdfsFileSystem.SCHEME + "://"
+ conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
fs = FileSystem.get(new URI(uri), conf);
defaultWorkingDirectory = fs.getWorkingDirectory().toUri().getPath();
}
@Override
protected String getDefaultWorkingDirectory() {
return defaultWorkingDirectory;
}
}

93
hadoop-mapreduce-project/CHANGES.txt
View File

@ -2,9 +2,29 @@ Hadoop MapReduce Change Log
Trunk (unreleased changes)
INCOMPATIBLE CHANGES
NEW FEATURES
MAPREDUCE-2669. Add new examples for Mean, Median, and Standard Deviation.
(Plamen Jeliazkov via shv)
IMPROVEMENTS
MAPREDUCE-2887 due to HADOOP-7524 Change RPC to allow multiple protocols including multuple versions of the same protocol (sanjay Radia)
MAPREDUCE-2887 due to HADOOP-7524 Change RPC to allow multiple protocols
including multuple versions of the same protocol (sanjay Radia)
MAPREDUCE-2934. MR portion of HADOOP-7607 - Simplify the RPC proxy cleanup
process (atm)
BUG FIXES
MAPREDUCE-2784. [Gridmix] Bug fixes in ExecutionSummarizer and
ResourceUsageMatcher. (amarrk)
MAPREDUCE-2978. Fixed test-patch to make Jenkins report correct number of
findBugs, correct links to findBugs artifacts and no links to the
artifacts when there are no warnings. (Tom White via vinodkv).
Release 0.23.0 - Unreleased
@ -248,6 +268,24 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2655. Add audit logs to ResourceManager and NodeManager. (Thomas
Graves via acmurthy)
MAPREDUCE-2864. Normalize configuration variable names for YARN. (Robert
Evans via acmurthy)
MAPREDUCE-2690. Web-page for FifoScheduler. (Eric Payne via acmurthy)
MAPREDUCE-2711. Update TestBlockPlacementPolicyRaid for the new namesystem
and block management APIs. (szetszwo)
MAPREDUCE-2933. Change allocate call to return ContainerStatus for
completed containers rather than Container. (acmurthy)
MAPREDUCE-2675. Reformat JobHistory Server main page to be more
useful. (Robert Joseph Evans via vinodkv).
MAPREDUCE-2896. Simplify all apis to in
org.apache.hadoop.yarn.api.records.* to be get/set only. Added javadocs to
all public records. (acmurthy)
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
@ -1221,6 +1259,59 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2948. Hadoop streaming test failure, post MR-2767 (mahadev)
MAPREDUCE-2908. Fix all findbugs warnings. (vinodkv via acmurthy)
MAPREDUCE-2942. TestNMAuditLogger.testNMAuditLoggerWithIP failing (Thomas Graves
via mahadev)
MAPREDUCE-2947. Fixed race condition in AuxiliaryServices. (vinodkv via
acmurthy)
MAPREDUCE-2844. Fixed display of nodes in UI. (Ravi Teja Ch N V via
acmurthy)
MAPREDUCE-2677. Fixed 404 for some links from HistoryServer. (Robert Evans
via acmurthy)
MAPREDUCE-2937. Ensure reason for application failure is displayed to the
user. (mahadev via acmurthy)
MAPREDUCE-2953. Fix a race condition on submission which caused client to
incorrectly assume application was gone by making submission synchronous
for RMAppManager. (Thomas Graves via acmurthy)
MAPREDUCE-2963. Fix hang in TestMRJobs. (Siddharth Seth via acmurthy)
MAPREDUCE-2954. Fixed a deadlock in NM caused due to wrong synchronization
in protocol buffer records. (Siddharth Seth via vinodkv)
MAPREDUCE-2975. Fixed YARNRunner to use YarnConfiguration rather than
Configuration. (mahadev via acmurthy)
MAPREDUCE-2971. ant build mapreduce fails protected access jc.displayJobList
(jobs) (Thomas Graves via mahadev)
MAPREDUCE-2691. Finishing up the cleanup of distributed cache file resources
and related tests. (Siddharth Seth via vinodkv)
MAPREDUCE-2749. Ensure NM registers with RM after starting all its services
correctly. (Thomas Graves via acmurthy)
MAPREDUCE-2979. Removed the needless ClientProtocolProvider configuration
from the hadoop-mapreduce-client-core module. (Siddharth Seth via vinodkv)
MAPREDUCE-2985. Fixed findbugs warnings in ResourceLocalizationService.
(Thomas Graves via acmurthy)
MAPREDUCE-2874. Fix formatting of ApplicationId in web-ui. (Eric Payne via
acmurthy)
MAPREDUCE-2995. Better handling of expired containers in MapReduce
ApplicationMaster. (vinodkv via acmurthy)
MAPREDUCE-2995. Fixed race condition in ContainerLauncher. (vinodkv via
acmurthy)
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES

4
hadoop-mapreduce-project/INSTALL
View File

@ -56,12 +56,12 @@ export YARN_CONF_DIR=$HADOOP_CONF_DIR
Step 9) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager.
<property>
<name>nodemanager.auxiluary.services</name>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce.shuffle</value>
</property>
<property>
<name>nodemanager.aux.service.mapreduce.shuffle.class</name>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>

16
hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
View File

@ -137,6 +137,11 @@
<Class name="org.apache.hadoop.mapred.Task$TaskReporter" />
<Method name="run" />
<Bug pattern="DM_EXIT" />
</Match>
<Match>
<Class name="org.apache.hadoop.mapreduce.security.token.DelegationTokenRenewal$DelegationTokenCancelThread" />
<Method name="run" />
<Bug pattern="DM_EXIT" />
</Match>
<!--
We need to cast objects between old and new api objects
@ -155,7 +160,8 @@
</Match>
<Match>
<Class name="org.apache.hadoop.mapred.FileOutputCommitter" />
<Bug pattern="NM_WRONG_PACKAGE_INTENTIONAL" />
<Method name="commitJob" />
<Bug pattern="NM_WRONG_PACKAGE" />
</Match>
<Match>
<Class name="org.apache.hadoop.mapred.OutputCommitter" />
@ -166,6 +172,14 @@
</Or>
<Bug pattern="NM_WRONG_PACKAGE_INTENTIONAL" />
</Match>
<Match>
<Class name="org.apache.hadoop.mapred.TaskCompletionEvent" />
<Or>
<Method name="setTaskStatus" />
<Method name="setTaskAttemptId" />
</Or>
<Bug pattern="NM_WRONG_PACKAGE" />
</Match>
<Match>
<Class name="org.apache.hadoop.mapred.lib.db.DBInputFormat$DBRecordReader" />
<Method name="next" />

2
hadoop-mapreduce-project/dev-support/test-patch.properties
View File

@ -14,5 +14,5 @@
# limitations under the License.
OK_RELEASEAUDIT_WARNINGS=2
OK_FINDBUGS_WARNINGS=13
OK_FINDBUGS_WARNINGS=0
OK_JAVADOC_WARNINGS=0

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
View File

@ -42,7 +42,6 @@
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.AMConstants;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunchedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
View File

@ -34,10 +34,10 @@
import org.apache.hadoop.ipc.RPC.Server;
import org.apache.hadoop.ipc.VersionedProtocol;
import org.apache.hadoop.mapred.SortedRanges.Range;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AMConstants;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler;
@ -104,8 +104,8 @@ protected void startRpcServer() {
try {
server =
RPC.getServer(TaskUmbilicalProtocol.class, this, "0.0.0.0", 0,
conf.getInt(AMConstants.AM_TASK_LISTENER_THREADS,
AMConstants.DEFAULT_AM_TASK_LISTENER_THREADS),
conf.getInt(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT,
MRJobConfig.DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT),
false, conf, jobTokenSecretManager);
server.start();
InetSocketAddress listenerAddress = server.getListenerAddress();

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/WrappedProgressSplitsBlock.java
View File

@ -20,9 +20,6 @@
// Workaround for ProgressSplitBlock being package access
public class WrappedProgressSplitsBlock extends ProgressSplitsBlock {
public static final int DEFAULT_NUMBER_PROGRESS_SPLITS = 12;
private WrappedPeriodicStatsAccumulator wrappedProgressWallclockTime;
private WrappedPeriodicStatsAccumulator wrappedProgressCPUTime;
private WrappedPeriodicStatsAccumulator wrappedProgressVirtualMemoryKbytes;

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
View File

@ -37,12 +37,12 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.Counter;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.security.UserGroupInformation;
@ -140,7 +140,7 @@ public void init(Configuration conf) {
LOG.info("Creating intermediate history logDir: ["
+ doneDirPath
+ "] + based on conf. Should ideally be created by the JobHistoryServer: "
+ JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY);
+ MRJobConfig.MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR);
mkdir(
doneDirFS,
doneDirPath,
@ -154,7 +154,7 @@ public void init(Configuration conf) {
String message = "Not creating intermediate history logDir: ["
+ doneDirPath
+ "] based on conf: "
+ JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY
+ MRJobConfig.MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR
+ ". Either set to true or pre-create this directory with appropriate permissions";
LOG.error(message);
throw new YarnException(message);

74
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AMConstants.java
View File

@ -1,74 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce.v2.app;
import org.apache.hadoop.mapreduce.v2.MRConstants;
public interface AMConstants {
public static final String CONTAINERLAUNCHER_THREADPOOL_SIZE =
"yarn.mapreduce.containerlauncher.threadpool-size";
public static final String AM_RM_SCHEDULE_INTERVAL =
"yarn.appMaster.scheduler.interval";
public static final int DEFAULT_AM_RM_SCHEDULE_INTERVAL = 2000;
public static final String AM_TASK_LISTENER_THREADS =
MRConstants.YARN_MR_PREFIX + "task.listener.threads";
public static final int DEFAULT_AM_TASK_LISTENER_THREADS = 10;
public static final String AM_JOB_CLIENT_THREADS =
MRConstants.YARN_MR_PREFIX + "job.client.threads";
public static final int DEFAULT_AM_JOB_CLIENT_THREADS = 1;
public static final String SPECULATOR_CLASS =
MRConstants.YARN_MR_PREFIX + "speculator.class";
public static final String TASK_RUNTIME_ESTIMATOR_CLASS =
MRConstants.YARN_MR_PREFIX + "task.runtime.estimator.class";
public static final String TASK_ATTEMPT_PROGRESS_RUNTIME_LINEARIZER_CLASS =
MRConstants.YARN_MR_PREFIX + "task.runtime.linearizer.class";
public static final String EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS =
MRConstants.YARN_MR_PREFIX
+ "task.runtime.estimator.exponential.smooth.lambda";
public static final String EXPONENTIAL_SMOOTHING_SMOOTH_RATE =
MRConstants.YARN_MR_PREFIX
+ "task.runtime.estimator.exponential.smooth.smoothsrate";
public static final String RECOVERY_ENABLE = MRConstants.YARN_MR_PREFIX
+ "recovery.enable";
public static final float DEFAULT_REDUCE_RAMP_UP_LIMIT = 0.5f;
public static final String REDUCE_RAMPUP_UP_LIMIT = MRConstants.YARN_MR_PREFIX
+ "reduce.rampup.limit";
public static final float DEFAULT_REDUCE_PREEMPTION_LIMIT = 0.5f;
public static final String REDUCE_PREEMPTION_LIMIT = MRConstants.YARN_MR_PREFIX
+ "reduce.preemption.limit";
public static final String NODE_BLACKLISTING_ENABLE = MRConstants.YARN_MR_PREFIX
+ "node.blacklisting.enable";
}

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
View File

@ -154,7 +154,7 @@ public void init(final Configuration conf) {
// for an app later
appName = conf.get(MRJobConfig.JOB_NAME, "<missing app name>");
if (conf.getBoolean(AMConstants.RECOVERY_ENABLE, false)
if (conf.getBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, false)
&& startCount > 1) {
LOG.info("Recovery is enabled. Will try to recover from previous life.");
Recovery recoveryServ = new RecoveryService(appID, clock, startCount);
@ -349,7 +349,7 @@ protected Speculator createSpeculator(Configuration conf, AppContext context) {
try {
speculatorClass
// "yarn.mapreduce.job.speculator.class"
= conf.getClass(AMConstants.SPECULATOR_CLASS,
= conf.getClass(MRJobConfig.MR_AM_JOB_SPECULATOR,
DefaultSpeculator.class,
Speculator.class);
Constructor<? extends Speculator> speculatorConstructor
@ -360,19 +360,19 @@ protected Speculator createSpeculator(Configuration conf, AppContext context) {
return result;
} catch (InstantiationException ex) {
LOG.error("Can't make a speculator -- check "
+ AMConstants.SPECULATOR_CLASS + " " + ex);
+ MRJobConfig.MR_AM_JOB_SPECULATOR, ex);
throw new YarnException(ex);
} catch (IllegalAccessException ex) {
LOG.error("Can't make a speculator -- check "
+ AMConstants.SPECULATOR_CLASS + " " + ex);
+ MRJobConfig.MR_AM_JOB_SPECULATOR, ex);
throw new YarnException(ex);
} catch (InvocationTargetException ex) {
LOG.error("Can't make a speculator -- check "
+ AMConstants.SPECULATOR_CLASS + " " + ex);
+ MRJobConfig.MR_AM_JOB_SPECULATOR, ex);
throw new YarnException(ex);
} catch (NoSuchMethodException ex) {
LOG.error("Can't make a speculator -- check "
+ AMConstants.SPECULATOR_CLASS + " " + ex);
+ MRJobConfig.MR_AM_JOB_SPECULATOR, ex);
throw new YarnException(ex);
}
}

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
View File

@ -32,6 +32,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
@ -59,7 +60,6 @@
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AMConstants;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
@ -141,8 +141,8 @@ public void start() {
server =
rpc.getServer(MRClientProtocol.class, protocolHandler, address,
conf, secretManager,
conf.getInt(AMConstants.AM_JOB_CLIENT_THREADS,
AMConstants.DEFAULT_AM_JOB_CLIENT_THREADS));
conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT,
MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT));
server.start();
this.bindAddress =
NetUtils.createSocketAddr(hostNameResolved.getHostAddress()

87
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
View File

@ -94,7 +94,6 @@
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
import org.apache.hadoop.mapreduce.v2.app.speculate.SpeculatorEvent;
import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleanupEvent;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
@ -138,6 +137,7 @@ public abstract class TaskAttemptImpl implements
protected final Configuration conf;
protected final Path jobFile;
protected final int partition;
@SuppressWarnings("rawtypes")
protected final EventHandler eventHandler;
private final TaskAttemptId attemptId;
private final Clock clock;
@ -204,6 +204,11 @@ TaskAttemptEventType.TA_FAILMSG, new DeallocateContainerTransition(
.addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.FAILED,
TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED,
new DeallocateContainerTransition(TaskAttemptState.FAILED, false))
.addTransition(TaskAttemptState.ASSIGNED,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
CLEANUP_CONTAINER_TRANSITION)
// ^ If RM kills the container due to expiry, preemption etc.
.addTransition(TaskAttemptState.ASSIGNED,
TaskAttemptState.KILL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_KILL, CLEANUP_CONTAINER_TRANSITION)
@ -432,7 +437,8 @@ TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
//this is the last status reported by the REMOTE running attempt
private TaskAttemptStatus reportedStatus;
public TaskAttemptImpl(TaskId taskId, int i, EventHandler eventHandler,
public TaskAttemptImpl(TaskId taskId, int i,
@SuppressWarnings("rawtypes") EventHandler eventHandler,
TaskAttemptListener taskAttemptListener, Path jobFile, int partition,
Configuration conf, String[] dataLocalHosts, OutputCommitter committer,
Token<JobTokenIdentifier> jobToken,
@ -528,6 +534,13 @@ private ContainerLaunchContext createContainerLaunchContext() {
ContainerLaunchContext container =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
// Application resources
Map<String, LocalResource> localResources =
new HashMap<String, LocalResource>();
// Application environment
Map<String, String> environment = new HashMap<String, String>();
try {
FileSystem remoteFS = FileSystem.get(conf);
@ -536,7 +549,7 @@ private ContainerLaunchContext createContainerLaunchContext() {
Path remoteJobJar = (new Path(remoteTask.getConf().get(
MRJobConfig.JAR))).makeQualified(remoteFS.getUri(),
remoteFS.getWorkingDirectory());
container.setLocalResource(
localResources.put(
MRConstants.JOB_JAR,
createLocalResource(remoteFS, recordFactory, remoteJobJar,
LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
@ -558,7 +571,7 @@ private ContainerLaunchContext createContainerLaunchContext() {
new Path(path, oldJobId.toString());
Path remoteJobConfPath =
new Path(remoteJobSubmitDir, MRConstants.JOB_CONF_FILE);
container.setLocalResource(
localResources.put(
MRConstants.JOB_CONF_FILE,
createLocalResource(remoteFS, recordFactory, remoteJobConfPath,
LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
@ -566,8 +579,13 @@ private ContainerLaunchContext createContainerLaunchContext() {
+ remoteJobConfPath.toUri().toASCIIString());
// //////////// End of JobConf setup
// Setup DistributedCache
setupDistributedCache(remoteFS, conf, container);
setupDistributedCache(remoteFS, conf, localResources, environment);
// Set local-resources and environment
container.setLocalResources(localResources);
container.setEnv(environment);
// Setup up tokens
Credentials taskCredentials = new Credentials();
@ -595,12 +613,12 @@ private ContainerLaunchContext createContainerLaunchContext() {
// Add shuffle token
LOG.info("Putting shuffle token in serviceData");
container
.setServiceData(
ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
ShuffleHandler.serializeServiceData(jobToken));
container.setServiceData(serviceData);
MRApps.addToClassPath(container.getAllEnv(), getInitialClasspath());
MRApps.addToClassPath(container.getEnv(), getInitialClasspath());
} catch (IOException e) {
throw new YarnException(e);
}
@ -623,11 +641,11 @@ private ContainerLaunchContext createContainerLaunchContext() {
classPaths.add(workDir.toString()); // TODO
// Construct the actual Container
container.addAllCommands(MapReduceChildJVM.getVMCommand(
container.setCommands(MapReduceChildJVM.getVMCommand(
taskAttemptListener.getAddress(), remoteTask, javaHome,
workDir.toString(), containerLogDir, childTmpDir, jvmID));
MapReduceChildJVM.setVMEnv(container.getAllEnv(), classPaths,
MapReduceChildJVM.setVMEnv(container.getEnv(), classPaths,
workDir.toString(), containerLogDir, nmLdLibraryPath, remoteTask,
localizedApplicationTokensFile);
@ -649,11 +667,15 @@ private static long[] parseTimeStamps(String[] strs) {
return result;
}
private void setupDistributedCache(FileSystem remoteFS, Configuration conf,
ContainerLaunchContext container) throws IOException {
private void setupDistributedCache(FileSystem remoteFS,
Configuration conf,
Map<String, LocalResource> localResources,
Map<String, String> env)
throws IOException {
// Cache archives
parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.ARCHIVE,
parseDistributedCacheArtifacts(remoteFS, localResources, env,
LocalResourceType.ARCHIVE,
DistributedCache.getCacheArchives(conf),
parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)),
getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES),
@ -661,7 +683,9 @@ private void setupDistributedCache(FileSystem remoteFS, Configuration conf,
DistributedCache.getArchiveClassPaths(conf));
// Cache files
parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.FILE,
parseDistributedCacheArtifacts(remoteFS,
localResources, env,
LocalResourceType.FILE,
DistributedCache.getCacheFiles(conf),
parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
@ -673,7 +697,10 @@ private void setupDistributedCache(FileSystem remoteFS, Configuration conf,
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
private void parseDistributedCacheArtifacts(
FileSystem remoteFS, ContainerLaunchContext container, LocalResourceType type,
FileSystem remoteFS,
Map<String, LocalResource> localResources,
Map<String, String> env,
LocalResourceType type,
URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[],
Path[] pathsToPutOnClasspath) throws IOException {
@ -710,9 +737,9 @@ private void parseDistributedCacheArtifacts(
throw new IllegalArgumentException("Resource name must be relative");
}
String linkName = name.toUri().getPath();
container.setLocalResource(
localResources.put(
linkName,
BuilderUtils.newLocalResource(recordFactory,
BuilderUtils.newLocalResource(
p.toUri(), type,
visibilities[i]
? LocalResourceVisibility.PUBLIC
@ -720,8 +747,7 @@ private void parseDistributedCacheArtifacts(
sizes[i], timestamps[i])
);
if (classPaths.containsKey(u.getPath())) {
Map<String, String> environment = container.getAllEnv();
MRApps.addToClassPath(environment, linkName);
MRApps.addToClassPath(env, linkName);
}
}
}
@ -893,6 +919,7 @@ public TaskAttemptState getState() {
}
}
@SuppressWarnings("unchecked")
@Override
public void handle(TaskAttemptEvent event) {
LOG.info("Processing " + event.getTaskAttemptID() +
@ -903,7 +930,8 @@ public void handle(TaskAttemptEvent event) {
try {
stateMachine.doTransition(event.getType(), event);
} catch (InvalidStateTransitonException e) {
LOG.error("Can't handle this event at current state", e);
LOG.error("Can't handle this event at current state for "
+ this.attemptId, e);
eventHandler.handle(new JobDiagnosticsUpdateEvent(
this.attemptId.getTaskId().getJobId(), "Invalid event " + event.getType() +
" on TaskAttempt " + this.attemptId));
@ -981,8 +1009,8 @@ private WrappedProgressSplitsBlock getProgressSplitBlock() {
try {
if (progressSplitBlock == null) {
progressSplitBlock = new WrappedProgressSplitsBlock(conf.getInt(
JHConfig.JOBHISTORY_TASKPROGRESS_NUMBER_SPLITS_KEY,
WrappedProgressSplitsBlock.DEFAULT_NUMBER_PROGRESS_SPLITS));
MRJobConfig.MR_AM_NUM_PROGRESS_SPLITS,
MRJobConfig.DEFAULT_MR_AM_NUM_PROGRESS_SPLITS));
}
return progressSplitBlock;
} finally {
@ -1035,6 +1063,7 @@ private static class RequestContainerTransition implements
public RequestContainerTransition(boolean rescheduled) {
this.rescheduled = rescheduled;
}
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
@ -1063,6 +1092,7 @@ public void transition(TaskAttemptImpl taskAttempt,
private static class ContainerAssignedTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(final TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
@ -1112,6 +1142,7 @@ private static class DeallocateContainerTransition implements
this.finalState = finalState;
this.withdrawsContainerRequest = withdrawsContainerRequest;
}
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
@ -1158,6 +1189,7 @@ public void transition(TaskAttemptImpl taskAttempt,
private static class LaunchedContainerTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent evnt) {
@ -1208,6 +1240,7 @@ public void transition(TaskAttemptImpl taskAttempt,
private static class CommitPendingTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
@ -1219,6 +1252,7 @@ public void transition(TaskAttemptImpl taskAttempt,
private static class TaskCleanupTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
@ -1234,6 +1268,7 @@ public void transition(TaskAttemptImpl taskAttempt,
private static class SucceededTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
@ -1263,6 +1298,7 @@ public void transition(TaskAttemptImpl taskAttempt,
private static class FailedTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
// set the finish time
@ -1287,6 +1323,7 @@ public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
}
}
@SuppressWarnings({ "unchecked" })
private void logAttemptFinishedEvent(TaskAttemptState state) {
//Log finished events only if an attempt started.
if (getLaunchTime() == 0) return;
@ -1320,6 +1357,7 @@ private void logAttemptFinishedEvent(TaskAttemptState state) {
private static class TooManyFetchFailureTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
//add to diagnostic
@ -1347,6 +1385,7 @@ public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
private static class KilledTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
@ -1373,6 +1412,7 @@ public void transition(TaskAttemptImpl taskAttempt,
private static class CleanupContainerTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
@ -1399,6 +1439,7 @@ private void addDiagnosticInfo(String diag) {
private static class StatusUpdater
implements SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
View File

@ -528,7 +528,8 @@ public void handle(TaskEvent event) {
try {
stateMachine.doTransition(event.getType(), event);
} catch (InvalidStateTransitonException e) {
LOG.error("Can't handle this event at current state", e);
LOG.error("Can't handle this event at current state for "
+ this.taskId, e);
internalError(event.getType());
}
if (oldState != getState()) {

35
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
View File

@ -21,9 +21,9 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@ -33,8 +33,8 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.app.AMConstants;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunchedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
@ -79,8 +79,8 @@ public class ContainerLauncherImpl extends AbstractService implements
private RecordFactory recordFactory;
//have a cache/map of UGIs so as to avoid creating too many RPC
//client connection objects to the same NodeManager
private Map<String, UserGroupInformation> ugiMap =
new HashMap<String, UserGroupInformation>();
private ConcurrentMap<String, UserGroupInformation> ugiMap =
new ConcurrentHashMap<String, UserGroupInformation>();
public ContainerLauncherImpl(AppContext context) {
super(ContainerLauncherImpl.class.getName());
@ -102,7 +102,7 @@ public synchronized void init(Configuration conf) {
public void start() {
launcherPool =
new ThreadPoolExecutor(getConfig().getInt(
AMConstants.CONTAINERLAUNCHER_THREADPOOL_SIZE, 10),
MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT, 10),
Integer.MAX_VALUE, 1, TimeUnit.HOURS,
new LinkedBlockingQueue<Runnable>());
launcherPool.prestartAllCoreThreads(); // Wait for work.
@ -142,23 +142,20 @@ protected ContainerManager getCMProxy(ContainerId containerID,
UserGroupInformation user = UserGroupInformation.getCurrentUser();
// TODO: Synchronization problems!!
if (UserGroupInformation.isSecurityEnabled()) {
if(!ugiMap.containsKey(containerManagerBindAddr)) {
Token<ContainerTokenIdentifier> token =
new Token<ContainerTokenIdentifier>(
containerToken.getIdentifier().array(),
containerToken.getPassword().array(), new Text(
containerToken.getKind()), new Text(
containerToken.getService()));
Token<ContainerTokenIdentifier> token = new Token<ContainerTokenIdentifier>(
containerToken.getIdentifier().array(), containerToken
.getPassword().array(), new Text(containerToken.getKind()),
new Text(containerToken.getService()));
// the user in createRemoteUser in this context is not important
user = UserGroupInformation.createRemoteUser(containerManagerBindAddr);
user.addToken(token);
ugiMap.put(containerManagerBindAddr, user);
} else {
UserGroupInformation ugi = UserGroupInformation
.createRemoteUser(containerManagerBindAddr);
ugi.addToken(token);
ugiMap.putIfAbsent(containerManagerBindAddr, ugi);
user = ugiMap.get(containerManagerBindAddr);
}
}
ContainerManager proxy =
user.doAs(new PrivilegedAction<ContainerManager>() {
@Override

Some files were not shown because too many files have changed in this diff Show More