big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6763. Shuffle server listen queue is too small (Jason Lowe via Varun Saxena)

Browse Source
This commit is contained in:
Varun Saxena 2016-08-20 03:58:01 +05:30
parent 763f0497bb
commit 723facfa40
2 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
View File

@ -210,6 +210,12 @@
</description> </description>
</property> </property>
<property>
<name>mapreduce.shuffle.listen.queue.size</name>
<value>128</value>
<description>The length of the shuffle server listen queue.</description>
</property>
<property> <property>
<name>mapreduce.shuffle.connection-keep-alive.enable</name> <name>mapreduce.shuffle.connection-keep-alive.enable</name>
<value>false</value> <value>false</value>

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
View File

@ -195,6 +195,10 @@ public class ShuffleHandler extends AuxiliaryService {
public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port"; public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port";
public static final int DEFAULT_SHUFFLE_PORT = 13562; public static final int DEFAULT_SHUFFLE_PORT = 13562;
public static final String SHUFFLE_LISTEN_QUEUE_SIZE =
"mapreduce.shuffle.listen.queue.size";
public static final int DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE = 128;
public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED =
"mapreduce.shuffle.connection-keep-alive.enable"; "mapreduce.shuffle.connection-keep-alive.enable";
public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false; public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false;
@ -504,6 +508,8 @@ protected void serviceStart() throws Exception {
} catch (Exception ex) { } catch (Exception ex) {
throw new RuntimeException(ex); throw new RuntimeException(ex);
} }
bootstrap.setOption("backlog", conf.getInt(SHUFFLE_LISTEN_QUEUE_SIZE,
DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE));
bootstrap.setOption("child.keepAlive", true); bootstrap.setOption("child.keepAlive", true);
bootstrap.setPipelineFactory(pipelineFact); bootstrap.setPipelineFactory(pipelineFact);
port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT); port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);