diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f57ba585d5..641aeb1912 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -27,6 +27,9 @@ Trunk (unreleased changes) HADOOP-7693. Enhance AvroRpcEngine to support the new #addProtocol interface introduced in HADOOP-7524. (cutting) + + HADOOP-7469 Add a standard handler for socket connection problems which + improves diagnostics (Uma Maheswara Rao G and stevel via stevel) BUGS diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index c339ce7eaa..58cf810186 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -23,8 +23,6 @@ import java.net.InetSocketAddress; import java.net.SocketTimeoutException; import java.net.UnknownHostException; -import java.net.ConnectException; - import java.io.IOException; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -235,8 +233,11 @@ public Connection(ConnectionId remoteId) throws IOException { this.remoteId = remoteId; this.server = remoteId.getAddress(); if (server.isUnresolved()) { - throw new UnknownHostException("unknown host: " + - remoteId.getAddress().getHostName()); + throw NetUtils.wrapException(remoteId.getAddress().getHostName(), + remoteId.getAddress().getPort(), + null, + 0, + new UnknownHostException()); } this.rpcTimeout = remoteId.getRpcTimeout(); this.maxIdleTime = remoteId.getMaxIdleTime(); @@ -1084,7 +1085,12 @@ public Writable call(Writable param, ConnectionId remoteId) call.error.fillInStackTrace(); throw call.error; } else { // local exception - throw wrapException(remoteId.getAddress(), call.error); + InetSocketAddress address = remoteId.getAddress(); + throw NetUtils.wrapException(address.getHostName(), + address.getPort(), + NetUtils.getHostname(), + 0, + call.error); } } else { return call.value; @@ -1093,37 +1099,6 @@ public Writable call(Writable param, ConnectionId remoteId) } /** - * Take an IOException and the address we were trying to connect to - * and return an IOException with the input exception as the cause. - * The new exception provides the stack trace of the place where - * the exception is thrown and some extra diagnostics information. - * If the exception is ConnectException or SocketTimeoutException, - * return a new one of the same type; Otherwise return an IOException. - * - * @param addr target address - * @param exception the relevant exception - * @return an exception to throw - */ - private IOException wrapException(InetSocketAddress addr, - IOException exception) { - if (exception instanceof ConnectException) { - //connection refused; include the host:port in the error - return (ConnectException)new ConnectException( - "Call to " + addr + " failed on connection exception: " + exception) - .initCause(exception); - } else if (exception instanceof SocketTimeoutException) { - return (SocketTimeoutException)new SocketTimeoutException( - "Call to " + addr + " failed on socket timeout exception: " - + exception).initCause(exception); - } else { - return (IOException)new IOException( - "Call to " + addr + " failed on local exception: " + exception) - .initCause(exception); - - } - } - - /** * @deprecated Use {@link #call(Writable[], InetSocketAddress[], * Class, UserGroupInformation, Configuration)} instead */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 0bfc5722f4..fa04120c5a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -51,8 +51,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import javax.security.sasl.Sasl; import javax.security.sasl.SaslException; @@ -70,6 +68,7 @@ import org.apache.hadoop.ipc.RPC.VersionMismatch; import org.apache.hadoop.ipc.metrics.RpcDetailedMetrics; import org.apache.hadoop.ipc.metrics.RpcMetrics; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; @@ -227,20 +226,11 @@ public static void bind(ServerSocket socket, InetSocketAddress address, int backlog) throws IOException { try { socket.bind(address, backlog); - } catch (BindException e) { - BindException bindException = new BindException("Problem binding to " + address - + " : " + e.getMessage()); - bindException.initCause(e); - throw bindException; } catch (SocketException e) { - // If they try to bind to a different host's address, give a better - // error message. - if ("Unresolved address".equals(e.getMessage())) { - throw new UnknownHostException("Invalid hostname for server: " + - address.getHostName()); - } else { - throw e; - } + throw NetUtils.wrapException(null, + 0, + address.getHostName(), + address.getPort(), e); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index d94b69f183..5f35b85b79 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -20,12 +20,15 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.BindException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.NetworkInterface; +import java.net.NoRouteToHostException; import java.net.Socket; import java.net.SocketAddress; import java.net.SocketException; +import java.net.SocketTimeoutException; import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; @@ -54,6 +57,13 @@ public class NetUtils { private static Map hostToResolved = new HashMap(); + /** text to point users elsewhere: {@value} */ + private static final String FOR_MORE_DETAILS_SEE + = " For more details see: "; + /** text included in wrapped exceptions if the host is null: {@value} */ + public static final String UNKNOWN_HOST = "(unknown)"; + /** Base URL of the Hadoop Wiki: {@value} */ + public static final String HADOOP_WIKI = "http://wiki.apache.org/hadoop/"; /** * Get the socket factory for the given class according to its @@ -537,4 +547,119 @@ public static boolean isLocalAddress(InetAddress addr) { } return local; } + + /** + * Take an IOException , the local host port and remote host port details and + * return an IOException with the input exception as the cause and also + * include the host details. The new exception provides the stack trace of the + * place where the exception is thrown and some extra diagnostics information. + * If the exception is BindException or ConnectException or + * UnknownHostException or SocketTimeoutException, return a new one of the + * same type; Otherwise return an IOException. + * + * @param destHost target host (nullable) + * @param destPort target port + * @param localHost local host (nullable) + * @param localPort local port + * @param exception the caught exception. + * @return an exception to throw + */ + public static IOException wrapException(final String destHost, + final int destPort, + final String localHost, + final int localPort, + final IOException exception) { + if (exception instanceof BindException) { + return new BindException( + "Problem binding to [" + + localHost + + ":" + + localPort + + "] " + + exception + + ";" + + see("BindException")); + } else if (exception instanceof ConnectException) { + // connection refused; include the host:port in the error + return (ConnectException) new ConnectException( + "Call From " + + localHost + + " to " + + destHost + + ":" + + destPort + + " failed on connection exception: " + + exception + + ";" + + see("ConnectionRefused")) + .initCause(exception); + } else if (exception instanceof UnknownHostException) { + return (UnknownHostException) new UnknownHostException( + "Invalid host name: " + + getHostDetailsAsString(destHost, destPort, localHost) + + exception + + ";" + + see("UnknownHost")) + .initCause(exception); + } else if (exception instanceof SocketTimeoutException) { + return (SocketTimeoutException) new SocketTimeoutException( + "Call From " + + localHost + " to " + destHost + ":" + destPort + + " failed on socket timeout exception: " + exception + + ";" + + see("SocketTimeout")) + .initCause(exception); + } else if (exception instanceof NoRouteToHostException) { + return (NoRouteToHostException) new NoRouteToHostException( + "No Route to Host from " + + localHost + " to " + destHost + ":" + destPort + + " failed on socket timeout exception: " + exception + + ";" + + see("NoRouteToHost")) + .initCause(exception); + } + else { + return (IOException) new IOException("Failed on local exception: " + + exception + + "; Host Details : " + + getHostDetailsAsString(destHost, destPort, localHost)) + .initCause(exception); + + } + } + + private static String see(final String entry) { + return FOR_MORE_DETAILS_SEE + HADOOP_WIKI + entry; + } + + /** + * Get the host details as a string + * @param destHost destinatioon host (nullable) + * @param destPort destination port + * @param localHost local host (nullable) + * @return a string describing the destination host:port and the local host + */ + private static String getHostDetailsAsString(final String destHost, + final int destPort, + final String localHost) { + StringBuilder hostDetails = new StringBuilder(27); + hostDetails.append("local host is: ") + .append(quoteHost(localHost)) + .append("; "); + hostDetails.append("destination host is: \"").append(quoteHost(destHost)) + .append(":") + .append(destPort).append("; "); + return hostDetails.toString(); + } + + /** + * Quote a hostname if it is not null + * @param hostname the hostname; nullable + * @return a quoted hostname or {@link #UNKNOWN_HOST} if the hostname is null + */ + private static String quoteHost(final String hostname) { + return (hostname != null) ? + ("\"" + hostname + "\"") + : UNKNOWN_HOST; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 5d04c20023..7c01e2f191 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -23,7 +23,6 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.net.NetUtils; @@ -270,7 +269,7 @@ public void testStandAloneClient() throws Exception { fail("Expected an exception to have been thrown"); } catch (IOException e) { String message = e.getMessage(); - String addressText = address.toString(); + String addressText = address.getHostName() + ":" + address.getPort(); assertTrue("Did not find "+addressText+" in "+message, message.contains(addressText)); Throwable cause=e.getCause(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java index 7cc6f4d521..d0927b9821 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java @@ -17,10 +17,15 @@ */ package org.apache.hadoop.net; +import junit.framework.AssertionFailedError; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.junit.Test; import static org.junit.Assert.*; +import java.io.IOException; +import java.net.BindException; import java.net.InetAddress; import java.net.NetworkInterface; import java.net.Socket; @@ -34,6 +39,12 @@ public class TestNetUtils { + private static final Log LOG = LogFactory.getLog(TestNetUtils.class); + private static final int DEST_PORT = 4040; + private static final String DEST_PORT_NAME = Integer.toString(DEST_PORT); + private static final int LOCAL_PORT = 8080; + private static final String LOCAL_PORT_NAME = Integer.toString(LOCAL_PORT); + /** * Test that we can't accidentally connect back to the connecting socket due * to a quirk in the TCP spec. @@ -120,4 +131,100 @@ public void testIsLocalAddress() throws Exception { } assertFalse(NetUtils.isLocalAddress(InetAddress.getByName("8.8.8.8"))); } + + @Test + public void testWrapConnectException() throws Throwable { + IOException e = new ConnectException("failed"); + IOException wrapped = verifyExceptionClass(e, ConnectException.class); + assertInException(wrapped, "failed"); + assertWikified(wrapped); + assertInException(wrapped, "localhost"); + assertRemoteDetailsIncluded(wrapped); + assertInException(wrapped, "/ConnectionRefused"); + } + + @Test + public void testWrapBindException() throws Throwable { + IOException e = new BindException("failed"); + IOException wrapped = verifyExceptionClass(e, BindException.class); + assertInException(wrapped, "failed"); + assertLocalDetailsIncluded(wrapped); + assertNotInException(wrapped, DEST_PORT_NAME); + assertInException(wrapped, "/BindException"); + } + + @Test + public void testWrapUnknownHostException() throws Throwable { + IOException e = new UnknownHostException("failed"); + IOException wrapped = verifyExceptionClass(e, UnknownHostException.class); + assertInException(wrapped, "failed"); + assertWikified(wrapped); + assertInException(wrapped, "localhost"); + assertRemoteDetailsIncluded(wrapped); + assertInException(wrapped, "/UnknownHost"); + } + + private void assertRemoteDetailsIncluded(IOException wrapped) + throws Throwable { + assertInException(wrapped, "desthost"); + assertInException(wrapped, DEST_PORT_NAME); + } + + private void assertLocalDetailsIncluded(IOException wrapped) + throws Throwable { + assertInException(wrapped, "localhost"); + assertInException(wrapped, LOCAL_PORT_NAME); + } + + private void assertWikified(Exception e) throws Throwable { + assertInException(e, NetUtils.HADOOP_WIKI); + } + + private void assertInException(Exception e, String text) throws Throwable { + String message = extractExceptionMessage(e); + if (!(message.contains(text))) { + throw new AssertionFailedError("Wrong text in message " + + "\"" + message + "\"" + + " expected \"" + text + "\"") + .initCause(e); + } + } + + private String extractExceptionMessage(Exception e) throws Throwable { + assertNotNull("Null Exception", e); + String message = e.getMessage(); + if (message == null) { + throw new AssertionFailedError("Empty text in exception " + e) + .initCause(e); + } + return message; + } + + private void assertNotInException(Exception e, String text) + throws Throwable{ + String message = extractExceptionMessage(e); + if (message.contains(text)) { + throw new AssertionFailedError("Wrong text in message " + + "\"" + message + "\"" + + " did not expect \"" + text + "\"") + .initCause(e); + } + } + + private IOException verifyExceptionClass(IOException e, + Class expectedClass) + throws Throwable { + assertNotNull("Null Exception", e); + IOException wrapped = + NetUtils.wrapException("desthost", DEST_PORT, + "localhost", LOCAL_PORT, + e); + LOG.info(wrapped.toString(), wrapped); + if(!(wrapped.getClass().equals(expectedClass))) { + throw new AssertionFailedError("Wrong exception class; expected " + + expectedClass + + " got " + wrapped.getClass() + ": " + wrapped).initCause(wrapped); + } + return wrapped; + } }