From 723facfa408f469891e67084bdafa18841a0cf64 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Sat, 20 Aug 2016 03:58:01 +0530 Subject: [PATCH] MAPREDUCE-6763. Shuffle server listen queue is too small (Jason Lowe via Varun Saxena) --- .../src/main/resources/mapred-default.xml | 6 ++++++ .../main/java/org/apache/hadoop/mapred/ShuffleHandler.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 33eece31c2..73aaa7a7db 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -210,6 +210,12 @@ + + mapreduce.shuffle.listen.queue.size + 128 + The length of the shuffle server listen queue. + + mapreduce.shuffle.connection-keep-alive.enable false diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index 8721ef534c..7818c8181c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -195,6 +195,10 @@ public class ShuffleHandler extends AuxiliaryService { public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port"; public static final int DEFAULT_SHUFFLE_PORT = 13562; + public static final String SHUFFLE_LISTEN_QUEUE_SIZE = + "mapreduce.shuffle.listen.queue.size"; + public static final int DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE = 128; + public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = "mapreduce.shuffle.connection-keep-alive.enable"; public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false; @@ -504,6 +508,8 @@ protected void serviceStart() throws Exception { } catch (Exception ex) { throw new RuntimeException(ex); } + bootstrap.setOption("backlog", conf.getInt(SHUFFLE_LISTEN_QUEUE_SIZE, + DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE)); bootstrap.setOption("child.keepAlive", true); bootstrap.setPipelineFactory(pipelineFact); port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);