big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7409. Make shuffle key length configurable. Contributed by Ashutosh Gupta.

Browse Source
This commit is contained in:
9uapaw 2022-08-31 17:30:24 +02:00
parent c69e16b297
commit 84081a8cae
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
View File

@ -71,7 +71,6 @@ class JobSubmitter {
protected static final Logger LOG = protected static final Logger LOG =
LoggerFactory.getLogger(JobSubmitter.class); LoggerFactory.getLogger(JobSubmitter.class);
private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1"; private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1";
private static final int SHUFFLE_KEY_LENGTH = 64;
private FileSystem jtFs; private FileSystem jtFs;
private ClientProtocol submitClient; private ClientProtocol submitClient;
private String submitHostName; private String submitHostName;
@ -177,7 +176,9 @@ JobStatus submitJobInternal(Job job, Cluster cluster)
KeyGenerator keyGen; KeyGenerator keyGen;
try { try {
keyGen = KeyGenerator.getInstance(SHUFFLE_KEYGEN_ALGORITHM); keyGen = KeyGenerator.getInstance(SHUFFLE_KEYGEN_ALGORITHM);
keyGen.init(SHUFFLE_KEY_LENGTH); int shuffleKeyLength =
conf.getInt(MRJobConfig.SHUFFLE_KEY_LENGTH, MRJobConfig.DEFAULT_SHUFFLE_KEY_LENGTH);
keyGen.init(shuffleKeyLength);
} catch (NoSuchAlgorithmException e) { } catch (NoSuchAlgorithmException e) {
throw new IOException("Error generating shuffle secret key", e); throw new IOException("Error generating shuffle secret key", e);
} }

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -1302,4 +1302,6 @@ public interface MRJobConfig {
*/ */
@Unstable @Unstable
String INPUT_FILE_MANDATORY_PREFIX = "mapreduce.job.input.file.must."; String INPUT_FILE_MANDATORY_PREFIX = "mapreduce.job.input.file.must.";
String SHUFFLE_KEY_LENGTH = "mapreduce.shuffle-key-length";
int DEFAULT_SHUFFLE_KEY_LENGTH = 64;
} }