HADOOP-6892. Common component of HDFS-1150 (Verify datanodes' identities to clients in secure clusters).
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@981688 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
666a8e1600
commit
1035138b4c
@ -30,6 +30,9 @@ Trunk (unreleased changes)
|
|||||||
file systems associated with a particular UGI. (Devaraj Das and Kan Zhang
|
file systems associated with a particular UGI. (Devaraj Das and Kan Zhang
|
||||||
via szetszwo)
|
via szetszwo)
|
||||||
|
|
||||||
|
HADOOP-6892. Common component of HDFS-1150 (Verify datanodes' identities
|
||||||
|
to clients in secure clusters) (jghoman)
|
||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
|
|
||||||
HADOOP-6644. util.Shell getGROUPS_FOR_USER_COMMAND method name
|
HADOOP-6644. util.Shell getGROUPS_FOR_USER_COMMAND method name
|
||||||
|
@ -77,20 +77,29 @@ if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
|
|||||||
. "${HADOOP_CONF_DIR}/hadoop-env.sh"
|
. "${HADOOP_CONF_DIR}/hadoop-env.sh"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Determine if we're starting a secure datanode, and if so, redefine appropriate variables
|
||||||
|
if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
|
||||||
|
export HADOOP_PID_DIR=$HADOOP_SECURE_DN_PID_DIR
|
||||||
|
export HADOOP_LOG_DIR=$HADOOP_SECURE_DN_LOG_DIR
|
||||||
|
export HADOOP_IDENT_STRING=$HADOOP_SECURE_DN_USER
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$HADOOP_IDENT_STRING" = "" ]; then
|
||||||
|
export HADOOP_IDENT_STRING="$USER"
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
# get log directory
|
# get log directory
|
||||||
if [ "$HADOOP_LOG_DIR" = "" ]; then
|
if [ "$HADOOP_LOG_DIR" = "" ]; then
|
||||||
export HADOOP_LOG_DIR="$HADOOP_HOME/logs"
|
export HADOOP_LOG_DIR="$HADOOP_HOME/logs"
|
||||||
fi
|
fi
|
||||||
mkdir -p "$HADOOP_LOG_DIR"
|
mkdir -p "$HADOOP_LOG_DIR"
|
||||||
|
chown $HADOOP_IDENT_STRING $HADOOP_LOG_DIR
|
||||||
|
|
||||||
if [ "$HADOOP_PID_DIR" = "" ]; then
|
if [ "$HADOOP_PID_DIR" = "" ]; then
|
||||||
HADOOP_PID_DIR=/tmp
|
HADOOP_PID_DIR=/tmp
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$HADOOP_IDENT_STRING" = "" ]; then
|
|
||||||
export HADOOP_IDENT_STRING="$USER"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# some variables
|
# some variables
|
||||||
export HADOOP_LOGFILE=hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.log
|
export HADOOP_LOGFILE=hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.log
|
||||||
export HADOOP_ROOT_LOGGER="INFO,DRFA"
|
export HADOOP_ROOT_LOGGER="INFO,DRFA"
|
||||||
|
@ -17,9 +17,9 @@
|
|||||||
*/
|
*/
|
||||||
package org.apache.hadoop.http;
|
package org.apache.hadoop.http;
|
||||||
|
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.PrintWriter;
|
import java.io.PrintWriter;
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
import java.net.BindException;
|
import java.net.BindException;
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
@ -43,17 +43,21 @@
|
|||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.conf.ConfServlet;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||||
|
import org.apache.hadoop.http.AdminAuthorizedServlet;
|
||||||
|
import org.apache.hadoop.http.FilterContainer;
|
||||||
|
import org.apache.hadoop.http.FilterInitializer;
|
||||||
|
import org.apache.hadoop.http.HtmlQuoting;
|
||||||
import org.apache.hadoop.log.LogLevel;
|
import org.apache.hadoop.log.LogLevel;
|
||||||
import org.apache.hadoop.metrics.MetricsServlet;
|
import org.apache.hadoop.metrics.MetricsServlet;
|
||||||
import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector;
|
import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector;
|
||||||
import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector.MODE;
|
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
|
import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector.MODE;
|
||||||
import org.apache.hadoop.security.authorize.AccessControlList;
|
import org.apache.hadoop.security.authorize.AccessControlList;
|
||||||
import org.apache.hadoop.util.ReflectionUtils;
|
import org.apache.hadoop.util.ReflectionUtils;
|
||||||
import org.apache.hadoop.conf.ConfServlet;
|
|
||||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
|
||||||
|
|
||||||
import org.mortbay.jetty.Connector;
|
import org.mortbay.jetty.Connector;
|
||||||
import org.mortbay.jetty.Handler;
|
import org.mortbay.jetty.Handler;
|
||||||
import org.mortbay.jetty.Server;
|
import org.mortbay.jetty.Server;
|
||||||
@ -103,12 +107,19 @@ public class HttpServer implements FilterContainer {
|
|||||||
static final String STATE_DESCRIPTION_ALIVE = " - alive";
|
static final String STATE_DESCRIPTION_ALIVE = " - alive";
|
||||||
static final String STATE_DESCRIPTION_NOT_LIVE = " - not live";
|
static final String STATE_DESCRIPTION_NOT_LIVE = " - not live";
|
||||||
|
|
||||||
|
private final boolean listenerStartedExternally;
|
||||||
|
|
||||||
/** Same as this(name, bindAddress, port, findPort, null); */
|
/** Same as this(name, bindAddress, port, findPort, null); */
|
||||||
public HttpServer(String name, String bindAddress, int port, boolean findPort
|
public HttpServer(String name, String bindAddress, int port, boolean findPort
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
this(name, bindAddress, port, findPort, new Configuration());
|
this(name, bindAddress, port, findPort, new Configuration());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public HttpServer(String name, String bindAddress, int port,
|
||||||
|
boolean findPort, Configuration conf, Connector connector) throws IOException {
|
||||||
|
this(name, bindAddress, port, findPort, conf, null, connector);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a status server on the given port.
|
* Create a status server on the given port.
|
||||||
* The jsp scripts are taken from src/webapps/<name>.
|
* The jsp scripts are taken from src/webapps/<name>.
|
||||||
@ -120,9 +131,15 @@ public HttpServer(String name, String bindAddress, int port, boolean findPort
|
|||||||
*/
|
*/
|
||||||
public HttpServer(String name, String bindAddress, int port,
|
public HttpServer(String name, String bindAddress, int port,
|
||||||
boolean findPort, Configuration conf) throws IOException {
|
boolean findPort, Configuration conf) throws IOException {
|
||||||
this(name, bindAddress, port, findPort, conf, null);
|
this(name, bindAddress, port, findPort, conf, null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public HttpServer(String name, String bindAddress, int port,
|
||||||
|
boolean findPort, Configuration conf, AccessControlList adminsAcl)
|
||||||
|
throws IOException {
|
||||||
|
this(name, bindAddress, port, findPort, conf, adminsAcl, null);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a status server on the given port.
|
* Create a status server on the given port.
|
||||||
* The jsp scripts are taken from src/webapps/<name>.
|
* The jsp scripts are taken from src/webapps/<name>.
|
||||||
@ -134,14 +151,22 @@ public HttpServer(String name, String bindAddress, int port,
|
|||||||
* @param adminsAcl {@link AccessControlList} of the admins
|
* @param adminsAcl {@link AccessControlList} of the admins
|
||||||
*/
|
*/
|
||||||
public HttpServer(String name, String bindAddress, int port,
|
public HttpServer(String name, String bindAddress, int port,
|
||||||
boolean findPort, Configuration conf, AccessControlList adminsAcl)
|
boolean findPort, Configuration conf, AccessControlList adminsAcl,
|
||||||
throws IOException {
|
Connector connector) throws IOException {
|
||||||
webServer = new Server();
|
webServer = new Server();
|
||||||
this.findPort = findPort;
|
this.findPort = findPort;
|
||||||
this.adminsAcl = adminsAcl;
|
this.adminsAcl = adminsAcl;
|
||||||
listener = createBaseListener(conf);
|
|
||||||
listener.setHost(bindAddress);
|
if(connector == null) {
|
||||||
listener.setPort(port);
|
listenerStartedExternally = false;
|
||||||
|
listener = createBaseListener(conf);
|
||||||
|
listener.setHost(bindAddress);
|
||||||
|
listener.setPort(port);
|
||||||
|
} else {
|
||||||
|
listenerStartedExternally = true;
|
||||||
|
listener = connector;
|
||||||
|
}
|
||||||
|
|
||||||
webServer.addConnector(listener);
|
webServer.addConnector(listener);
|
||||||
|
|
||||||
int maxThreads = conf.getInt(HTTP_MAX_THREADS, -1);
|
int maxThreads = conf.getInt(HTTP_MAX_THREADS, -1);
|
||||||
@ -184,8 +209,12 @@ public HttpServer(String name, String bindAddress, int port,
|
|||||||
* provided. This wrapper and all subclasses must create at least one
|
* provided. This wrapper and all subclasses must create at least one
|
||||||
* listener.
|
* listener.
|
||||||
*/
|
*/
|
||||||
protected Connector createBaseListener(Configuration conf)
|
public Connector createBaseListener(Configuration conf) throws IOException {
|
||||||
throws IOException {
|
return HttpServer.createDefaultChannelConnector();
|
||||||
|
}
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public static Connector createDefaultChannelConnector() {
|
||||||
SelectChannelConnector ret = new SelectChannelConnector();
|
SelectChannelConnector ret = new SelectChannelConnector();
|
||||||
ret.setLowResourceMaxIdleTime(10000);
|
ret.setLowResourceMaxIdleTime(10000);
|
||||||
ret.setAcceptQueueSize(128);
|
ret.setAcceptQueueSize(128);
|
||||||
@ -532,72 +561,80 @@ else if (!needCertsAuth && needKrbAuth)
|
|||||||
*/
|
*/
|
||||||
public void start() throws IOException {
|
public void start() throws IOException {
|
||||||
try {
|
try {
|
||||||
int port = 0;
|
if(listenerStartedExternally) { // Expect that listener was started securely
|
||||||
int oriPort = listener.getPort(); // The original requested port
|
if(listener.getLocalPort() == -1) // ... and verify
|
||||||
while (true) {
|
throw new Exception("Exepected webserver's listener to be started " +
|
||||||
try {
|
"previously but wasn't");
|
||||||
port = webServer.getConnectors()[0].getLocalPort();
|
// And skip all the port rolling issues.
|
||||||
LOG.info("Port returned by webServer.getConnectors()[0]." +
|
webServer.start();
|
||||||
"getLocalPort() before open() is "+ port +
|
} else {
|
||||||
". Opening the listener on " + oriPort);
|
int port = 0;
|
||||||
listener.open();
|
int oriPort = listener.getPort(); // The original requested port
|
||||||
port = listener.getLocalPort();
|
while (true) {
|
||||||
LOG.info("listener.getLocalPort() returned " + listener.getLocalPort() +
|
try {
|
||||||
" webServer.getConnectors()[0].getLocalPort() returned " +
|
port = webServer.getConnectors()[0].getLocalPort();
|
||||||
webServer.getConnectors()[0].getLocalPort());
|
LOG.info("Port returned by webServer.getConnectors()[0]." +
|
||||||
//Workaround to handle the problem reported in HADOOP-4744
|
"getLocalPort() before open() is "+ port +
|
||||||
if (port < 0) {
|
". Opening the listener on " + oriPort);
|
||||||
Thread.sleep(100);
|
listener.open();
|
||||||
int numRetries = 1;
|
port = listener.getLocalPort();
|
||||||
while (port < 0) {
|
LOG.info("listener.getLocalPort() returned " + listener.getLocalPort() +
|
||||||
LOG.warn("listener.getLocalPort returned " + port);
|
" webServer.getConnectors()[0].getLocalPort() returned " +
|
||||||
if (numRetries++ > MAX_RETRIES) {
|
webServer.getConnectors()[0].getLocalPort());
|
||||||
throw new Exception(" listener.getLocalPort is returning " +
|
//Workaround to handle the problem reported in HADOOP-4744
|
||||||
"less than 0 even after " +numRetries+" resets");
|
if (port < 0) {
|
||||||
}
|
Thread.sleep(100);
|
||||||
for (int i = 0; i < 2; i++) {
|
int numRetries = 1;
|
||||||
LOG.info("Retrying listener.getLocalPort()");
|
while (port < 0) {
|
||||||
port = listener.getLocalPort();
|
LOG.warn("listener.getLocalPort returned " + port);
|
||||||
|
if (numRetries++ > MAX_RETRIES) {
|
||||||
|
throw new Exception(" listener.getLocalPort is returning " +
|
||||||
|
"less than 0 even after " +numRetries+" resets");
|
||||||
|
}
|
||||||
|
for (int i = 0; i < 2; i++) {
|
||||||
|
LOG.info("Retrying listener.getLocalPort()");
|
||||||
|
port = listener.getLocalPort();
|
||||||
|
if (port > 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Thread.sleep(200);
|
||||||
|
}
|
||||||
if (port > 0) {
|
if (port > 0) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Thread.sleep(200);
|
LOG.info("Bouncing the listener");
|
||||||
|
listener.close();
|
||||||
|
Thread.sleep(1000);
|
||||||
|
listener.setPort(oriPort == 0 ? 0 : (oriPort += 1));
|
||||||
|
listener.open();
|
||||||
|
Thread.sleep(100);
|
||||||
|
port = listener.getLocalPort();
|
||||||
}
|
}
|
||||||
if (port > 0) {
|
} //Workaround end
|
||||||
break;
|
LOG.info("Jetty bound to port " + port);
|
||||||
|
webServer.start();
|
||||||
|
break;
|
||||||
|
} catch (IOException ex) {
|
||||||
|
// if this is a bind exception,
|
||||||
|
// then try the next port number.
|
||||||
|
if (ex instanceof BindException) {
|
||||||
|
if (!findPort) {
|
||||||
|
BindException be = new BindException(
|
||||||
|
"Port in use: " + listener.getHost()
|
||||||
|
+ ":" + listener.getPort());
|
||||||
|
be.initCause(ex);
|
||||||
|
throw be;
|
||||||
}
|
}
|
||||||
LOG.info("Bouncing the listener");
|
} else {
|
||||||
listener.close();
|
LOG.info("HttpServer.start() threw a non Bind IOException");
|
||||||
Thread.sleep(1000);
|
throw ex;
|
||||||
listener.setPort(oriPort == 0 ? 0 : (oriPort += 1));
|
|
||||||
listener.open();
|
|
||||||
Thread.sleep(100);
|
|
||||||
port = listener.getLocalPort();
|
|
||||||
}
|
}
|
||||||
} //Workaround end
|
} catch (MultiException ex) {
|
||||||
LOG.info("Jetty bound to port " + port);
|
LOG.info("HttpServer.start() threw a MultiException");
|
||||||
webServer.start();
|
|
||||||
break;
|
|
||||||
} catch (IOException ex) {
|
|
||||||
// if this is a bind exception,
|
|
||||||
// then try the next port number.
|
|
||||||
if (ex instanceof BindException) {
|
|
||||||
if (!findPort) {
|
|
||||||
BindException be = new BindException(
|
|
||||||
"Port in use: " + listener.getHost()
|
|
||||||
+ ":" + listener.getPort());
|
|
||||||
be.initCause(ex);
|
|
||||||
throw be;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
LOG.info("HttpServer.start() threw a non Bind IOException");
|
|
||||||
throw ex;
|
throw ex;
|
||||||
}
|
}
|
||||||
} catch (MultiException ex) {
|
listener.setPort((oriPort += 1));
|
||||||
LOG.info("HttpServer.start() threw a MultiException");
|
|
||||||
throw ex;
|
|
||||||
}
|
}
|
||||||
listener.setPort((oriPort += 1));
|
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw e;
|
throw e;
|
||||||
|
Loading…
Reference in New Issue
Block a user