big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5551. Ignore file backed pages from memory computation when smaps is enabled. Contributed by Rajesh Balamohan

Browse Source
This commit is contained in:
Jason Lowe 2016-10-11 15:12:43 +00:00
parent 96b12662ea
commit ecb51b857a
2 changed files with 39 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
View File

@ -406,15 +406,14 @@ private long getSmapBasedRssMemorySize(int olderThanAge) {
continue; continue;
} }
total += // Account for anonymous to know the amount of
Math.min(info.sharedDirty, info.pss) + info.privateDirty // memory reclaimable by killing the process
+ info.privateClean; total += info.anonymous;
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid() LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid()
+ ", SharedDirty : " + info.sharedDirty + ", PSS : " + ", info : " + info.toString()
+ info.pss + ", Private_Dirty : " + info.privateDirty + ", total : " + (total * KB_TO_BYTES));
+ ", Private_Clean : " + info.privateClean + ", total : "
+ (total * KB_TO_BYTES));
} }
} }
} }
@ -877,6 +876,7 @@ static class ProcessSmapMemoryInfo {
private int sharedDirty; private int sharedDirty;
private int privateClean; private int privateClean;
private int privateDirty; private int privateDirty;
private int anonymous;
private int referenced; private int referenced;
private String regionName; private String regionName;
private String permission; private String permission;
@ -929,6 +929,10 @@ public int getReferenced() {
return referenced; return referenced;
} }
public int getAnonymous() {
return anonymous;
}
public void setMemInfo(String key, String value) { public void setMemInfo(String key, String value) {
MemInfo info = MemInfo.getMemInfoByName(key); MemInfo info = MemInfo.getMemInfoByName(key);
int val = 0; int val = 0;
@ -969,6 +973,9 @@ public void setMemInfo(String key, String value) {
case REFERENCED: case REFERENCED:
referenced = val; referenced = val;
break; break;
case ANONYMOUS:
anonymous = val;
break;
default: default:
break; break;
} }
@ -999,10 +1006,7 @@ public String toString() {
.append(MemInfo.REFERENCED.name + ":" + this.getReferenced()) .append(MemInfo.REFERENCED.name + ":" + this.getReferenced())
.append(" kB\n"); .append(" kB\n");
sb.append("\t") sb.append("\t")
.append(MemInfo.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty()) .append(MemInfo.ANONYMOUS.name + ":" + this.getAnonymous())
.append(" kB\n");
sb.append("\t")
.append(MemInfo.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty())
.append(" kB\n"); .append(" kB\n");
return sb.toString(); return sb.toString();
} }

46
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
View File

@ -369,21 +369,24 @@ public void createMemoryMappingInfo(ProcessTreeSmapMemInfo[] procMemInfo) {
List<ProcessSmapMemoryInfo> memoryMappingList = List<ProcessSmapMemoryInfo> memoryMappingList =
procMemInfo[i].getMemoryInfoList(); procMemInfo[i].getMemoryInfoList();
memoryMappingList.add(constructMemoryMappingInfo( memoryMappingList.add(constructMemoryMappingInfo(
"7f56c177c000-7f56c177d000 " "7f56c177c000-7f56c177d000 "
+ "rw-p 00010000 08:02 40371558 " + "rw-p 00010000 08:02 40371558 "
+ "/grid/0/jdk1.7.0_25/jre/lib/amd64/libnio.so", + "/grid/0/jdk1.7.0_25/jre/lib/amd64/libnio.so",
new String[] { "4", "4", "25", "4", "25", "15", "10", "4", "0", "0", // Format: size, rss, pss, shared_clean, shared_dirty, private_clean
"0", "4", "4" })); // private_dirty, referenced, anon, anon-huge-pages, swap,
// kernel_page_size, mmu_page_size
new String[] {"4", "4", "25", "4", "25", "15", "10", "4", "10", "0",
"0", "4", "4"}));
memoryMappingList.add(constructMemoryMappingInfo( memoryMappingList.add(constructMemoryMappingInfo(
"7fb09382e000-7fb09382f000 r--s 00003000 " + "08:02 25953545", "7fb09382e000-7fb09382f000 r--s 00003000 " + "08:02 25953545",
new String[] { "4", "4", "25", "4", "0", "15", "10", "4", "0", "0", new String[] {"4", "4", "25", "4", "0", "15", "10", "4", "10", "0",
"0", "4", "4" })); "0", "4", "4"}));
memoryMappingList.add(constructMemoryMappingInfo( memoryMappingList.add(constructMemoryMappingInfo(
"7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] { "4", "4", "7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] {"4", "4",
"25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4" })); "25", "4", "0", "15", "10", "4", "10", "0", "0", "4", "4"}));
memoryMappingList.add(constructMemoryMappingInfo( memoryMappingList.add(constructMemoryMappingInfo(
"7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] { "4", "4", "7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] {"4", "4",
"25", "4", "50", "15", "10", "4", "0", "0", "0", "4", "4" })); "25", "4", "50", "15", "10", "4", "10", "0", "0", "4", "4"}));
} }
} }
@ -471,13 +474,12 @@ public void testCpuAndMemoryForProcessTree() throws IOException {
// Check by enabling smaps // Check by enabling smaps
setSmapsInProceTree(processTree, true); setSmapsInProceTree(processTree, true);
// RSS=Min(shared_dirty,PSS)+PrivateClean+PrivateDirty (exclude r-xs, // anon (exclude r-xs,r--s)
// r--s)
Assert.assertEquals("rss memory does not match", Assert.assertEquals("rss memory does not match",
(100 * KB_TO_BYTES * 3), processTree.getRssMemorySize()); (20 * KB_TO_BYTES * 3), processTree.getRssMemorySize());
// verify old API // verify old API
Assert.assertEquals("rss memory (old API) does not match", Assert.assertEquals("rss memory (old API) does not match",
(100 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem()); (20 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem());
// test the cpu time again to see if it cumulates // test the cpu time again to see if it cumulates
procInfos[0] = procInfos[0] =
@ -621,10 +623,10 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
cumuRssMem, processTree.getCumulativeRssmem()); cumuRssMem, processTree.getCumulativeRssmem());
} else { } else {
Assert.assertEquals("rssmem does not include new process", Assert.assertEquals("rssmem does not include new process",
100 * KB_TO_BYTES * 4, processTree.getRssMemorySize()); 20 * KB_TO_BYTES * 4, processTree.getRssMemorySize());
// verify old API // verify old API
Assert.assertEquals("rssmem (old API) does not include new process", Assert.assertEquals("rssmem (old API) does not include new process",
100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem()); 20 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem());
} }
// however processes older than 1 iteration will retain the older value // however processes older than 1 iteration will retain the older value
@ -650,11 +652,11 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
} else { } else {
Assert.assertEquals( Assert.assertEquals(
"rssmem shouldn't have included new process", "rssmem shouldn't have included new process",
100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1)); 20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1));
// Verify old API // Verify old API
Assert.assertEquals( Assert.assertEquals(
"rssmem (old API) shouldn't have included new process", "rssmem (old API) shouldn't have included new process",
100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1)); 20 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1));
} }
// one more process // one more process
@ -696,11 +698,11 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
} else { } else {
Assert.assertEquals( Assert.assertEquals(
"rssmem shouldn't have included new processes", "rssmem shouldn't have included new processes",
100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2)); 20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2));
// Verify old API // Verify old API
Assert.assertEquals( Assert.assertEquals(
"rssmem (old API) shouldn't have included new processes", "rssmem (old API) shouldn't have included new processes",
100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2)); 20 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2));
} }
// processes older than 1 iteration should not include new process, // processes older than 1 iteration should not include new process,
@ -727,10 +729,10 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
} else { } else {
Assert.assertEquals( Assert.assertEquals(
"rssmem shouldn't have included new processes", "rssmem shouldn't have included new processes",
100 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1)); 20 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1));
Assert.assertEquals( Assert.assertEquals(
"rssmem (old API) shouldn't have included new processes", "rssmem (old API) shouldn't have included new processes",
100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1)); 20 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1));
} }
// no processes older than 3 iterations // no processes older than 3 iterations