diff --git a/CHANGES.txt b/CHANGES.txt index a5941588d2..50b800ced1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -54,10 +54,14 @@ Trunk (unreleased changes) HADOOP-6510. Adds a way for superusers to impersonate other users in a secure environment. (Jitendra Nath Pandey via ddas) - HADOOP-6421 Adds Symbolic links to FileContext, AbstractFileSystem. + HADOOP-6421. Adds Symbolic links to FileContext, AbstractFileSystem. It also adds a limited implementation for the local file system (RawLocalFs) that allows local symlinks. (Eli Collins via Sanjay Radia) + HADOOP-6577. Add hidden configuration option "ipc.server.max.response.size" + to change the default 1 MB, the maximum size when large IPC handler + response buffer is reset. (suresh) + IMPROVEMENTS HADOOP-6283. Improve the exception messages thrown by diff --git a/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index e887231272..b33aa7b308 100644 --- a/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -119,6 +119,10 @@ public class CommonConfigurationKeys { public static final int IPC_CLIENT_IDLETHRESHOLD_DEFAULT = 4000; public static final String IPC_SERVER_TCPNODELAY_KEY = "ipc.server.tcpnodelay"; public static final boolean IPC_SERVER_TCPNODELAY_DEFAULT = false; + public static final String IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY = + "ipc.server.max.response.size"; + public static final int IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT = + 1024*1024; public static final String HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY = "hadoop.rpc.socket.factory.class.default"; diff --git a/src/java/org/apache/hadoop/ipc/Server.java b/src/java/org/apache/hadoop/ipc/Server.java index b338b00283..6297ec7a94 100644 --- a/src/java/org/apache/hadoop/ipc/Server.java +++ b/src/java/org/apache/hadoop/ipc/Server.java @@ -59,6 +59,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import static org.apache.hadoop.fs.CommonConfigurationKeys.*; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.ipc.metrics.RpcMetrics; @@ -104,8 +105,7 @@ public abstract class Server { * Initial and max size of response buffer */ static int INITIAL_RESP_BUF_SIZE = 10240; - static int MAX_RESP_BUF_SIZE = 1024*1024; - + public static final Log LOG = LogFactory.getLog(Server.class); private static final ThreadLocal SERVER = new ThreadLocal(); @@ -174,6 +174,7 @@ public static String getRemoteAddress() { private SecretManager secretManager; private int maxQueueSize; + private final int maxRespSize; private int socketSendBufferSize; private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm @@ -1207,7 +1208,7 @@ public Writable run() throws Exception { : Status.ERROR, value, errorClass, error); // Discard the large buf and reset it back to // smaller size to freeup heap - if (buf.size() > MAX_RESP_BUF_SIZE) { + if (buf.size() > maxRespSize) { LOG.warn("Large response size " + buf.size() + " for call " + call.toString()); buf = new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE); @@ -1253,6 +1254,8 @@ protected Server(String bindAddress, int port, this.handlerCount = handlerCount; this.socketSendBufferSize = 0; this.maxQueueSize = handlerCount * MAX_QUEUE_SIZE_PER_HANDLER; + this.maxRespSize = conf.getInt(IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY, + IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT); this.callQueue = new LinkedBlockingQueue(maxQueueSize); this.maxIdleTime = 2*conf.getInt("ipc.client.connection.maxidletime", 1000); this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10); diff --git a/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java b/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java index e1370fe866..3710198295 100644 --- a/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java +++ b/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.net.NetUtils; @@ -116,8 +117,10 @@ public void run() { public void testResponseBuffer() throws Exception { Server.INITIAL_RESP_BUF_SIZE = 1; - Server.MAX_RESP_BUF_SIZE = 1; + conf.setInt(CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY, + 1); testServerResponder(1, true, 1, 1, 5); + conf = new Configuration(); // reset configuration } public void testServerResponder() throws Exception {