big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-4301. Dedupe some strings in MRAM for memory savings (bobby via tgraves)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1346002 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas Graves 2012-06-04 15:14:27 +00:00
parent 30807fec82
commit 1283ab51a9
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -377,6 +377,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4162. Correctly set token service (Daryn Sharp via bobby) MAPREDUCE-4162. Correctly set token service (Daryn Sharp via bobby)
MAPREDUCE-4301. Dedupe some strings in MRAM for memory savings
(bobby via tgraves)
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReader.java
View File

@ -47,6 +47,7 @@ public static JobSplit.TaskSplitMetaInfo[] readSplitMetaInfo(
long maxMetaInfoSize = conf.getLong(JTConfig.JT_MAX_JOB_SPLIT_METAINFO_SIZE, long maxMetaInfoSize = conf.getLong(JTConfig.JT_MAX_JOB_SPLIT_METAINFO_SIZE,
10000000L); 10000000L);
Path metaSplitFile = JobSubmissionFiles.getJobSplitMetaFile(jobSubmitDir); Path metaSplitFile = JobSubmissionFiles.getJobSplitMetaFile(jobSubmitDir);
String jobSplitFile = JobSubmissionFiles.getJobSplitFile(jobSubmitDir).toString();
FileStatus fStatus = fs.getFileStatus(metaSplitFile); FileStatus fStatus = fs.getFileStatus(metaSplitFile);
if (maxMetaInfoSize > 0 && fStatus.getLen() > maxMetaInfoSize) { if (maxMetaInfoSize > 0 && fStatus.getLen() > maxMetaInfoSize) {
throw new IOException("Split metadata size exceeded " + throw new IOException("Split metadata size exceeded " +
@ -70,7 +71,7 @@ public static JobSplit.TaskSplitMetaInfo[] readSplitMetaInfo(
JobSplit.SplitMetaInfo splitMetaInfo = new JobSplit.SplitMetaInfo(); JobSplit.SplitMetaInfo splitMetaInfo = new JobSplit.SplitMetaInfo();
splitMetaInfo.readFields(in); splitMetaInfo.readFields(in);
JobSplit.TaskSplitIndex splitIndex = new JobSplit.TaskSplitIndex( JobSplit.TaskSplitIndex splitIndex = new JobSplit.TaskSplitIndex(
JobSubmissionFiles.getJobSplitFile(jobSubmitDir).toString(), jobSplitFile,
splitMetaInfo.getStartOffset()); splitMetaInfo.getStartOffset());
allSplitMetaInfo[i] = new JobSplit.TaskSplitMetaInfo(splitIndex, allSplitMetaInfo[i] = new JobSplit.TaskSplitMetaInfo(splitIndex,
splitMetaInfo.getLocations(), splitMetaInfo.getLocations(),