big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6343. JobConf.parseMaximumHeapSizeMB() fails to parse value greater than 2GB expressed in bytes. (Hao Xia via kasha)

Browse Source
This commit is contained in:
Karthik Kambatla 2015-04-28 14:05:26 -07:00
parent bc1bd7e5c4
commit 519092322d
3 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -188,6 +188,9 @@ Trunk (Unreleased)
MAPREDUCE-6234. TestHighRamJob fails due to the change in MAPREDUCE-5785.
(Masatake Iwasaki via kasha)
MAPREDUCE-6343. JobConf.parseMaximumHeapSizeMB() fails to parse value
greater than 2GB expressed in bytes. (Hao Xia via kasha)
BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
View File

@ -2080,28 +2080,28 @@ public static int parseMaximumHeapSizeMB(String javaOpts) {
// Find the last matching -Xmx following word boundaries
Matcher m = JAVA_OPTS_XMX_PATTERN.matcher(javaOpts);
if (m.matches()) {
int size = Integer.parseInt(m.group(1));
long size = Long.parseLong(m.group(1));
if (size <= 0) {
return -1;
}
if (m.group(2).isEmpty()) {
// -Xmx specified in bytes
return size / (1024 * 1024);
return (int) (size / (1024 * 1024));
}
char unit = m.group(2).charAt(0);
switch (unit) {
case 'g':
case 'G':
// -Xmx specified in GB
return size * 1024;
return (int) (size * 1024);
case 'm':
case 'M':
// -Xmx specified in MB
return size;
return (int) size;
case 'k':
case 'K':
// -Xmx specified in KB
return size / 1024;
return (int) (size / 1024);
}
}
// -Xmx not specified

16
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
View File

@ -361,4 +361,20 @@ public void testMaxTaskFailuresPerTracker() {
jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxReduceAttempts()
);
}
/**
* Test parsing various types of Java heap options.
*/
@Test
public void testParseMaximumHeapSizeMB() {
// happy cases
Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4294967296"));
Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4194304k"));
Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4096m"));
Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4g"));
// sad cases
Assert.assertEquals(-1, JobConf.parseMaximumHeapSizeMB("-Xmx4?"));
Assert.assertEquals(-1, JobConf.parseMaximumHeapSizeMB(""));
}
}