From 84081a8cae7d050cc3dc2b3bddd298d769032120 Mon Sep 17 00:00:00 2001 From: 9uapaw Date: Wed, 31 Aug 2022 17:30:24 +0200 Subject: [PATCH] MAPREDUCE-7409. Make shuffle key length configurable. Contributed by Ashutosh Gupta. --- .../main/java/org/apache/hadoop/mapreduce/JobSubmitter.java | 5 +++-- .../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java index 4c983178a7..232b8bf13b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java @@ -71,7 +71,6 @@ class JobSubmitter { protected static final Logger LOG = LoggerFactory.getLogger(JobSubmitter.class); private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1"; - private static final int SHUFFLE_KEY_LENGTH = 64; private FileSystem jtFs; private ClientProtocol submitClient; private String submitHostName; @@ -177,7 +176,9 @@ JobStatus submitJobInternal(Job job, Cluster cluster) KeyGenerator keyGen; try { keyGen = KeyGenerator.getInstance(SHUFFLE_KEYGEN_ALGORITHM); - keyGen.init(SHUFFLE_KEY_LENGTH); + int shuffleKeyLength = + conf.getInt(MRJobConfig.SHUFFLE_KEY_LENGTH, MRJobConfig.DEFAULT_SHUFFLE_KEY_LENGTH); + keyGen.init(shuffleKeyLength); } catch (NoSuchAlgorithmException e) { throw new IOException("Error generating shuffle secret key", e); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 15d57a6746..4523565f30 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -1302,4 +1302,6 @@ public interface MRJobConfig { */ @Unstable String INPUT_FILE_MANDATORY_PREFIX = "mapreduce.job.input.file.must."; + String SHUFFLE_KEY_LENGTH = "mapreduce.shuffle-key-length"; + int DEFAULT_SHUFFLE_KEY_LENGTH = 64; }