YARN-5551. Ignore file backed pages from memory computation when smaps is enabled. Contributed by Rajesh Balamohan

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java

@@ -406,15 +406,14 @@ private long getSmapBasedRssMemorySize(int olderThanAge) {
                 continue;
               }
 
-              total +=
+              // Account for anonymous to know the amount of
-                  Math.min(info.sharedDirty, info.pss) + info.privateDirty
+              // memory reclaimable by killing the process
-                      + info.privateClean;
+              total += info.anonymous;
+
               if (LOG.isDebugEnabled()) {
                 LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid()
-                    + ", SharedDirty : " + info.sharedDirty + ", PSS : "
+                    + ", info : " + info.toString()
-                    + info.pss + ", Private_Dirty : " + info.privateDirty
+                    + ", total : " + (total * KB_TO_BYTES));
-                    + ", Private_Clean : " + info.privateClean + ", total : "
-                    + (total * KB_TO_BYTES));
               }
             }
           }
@@ -877,6 +876,7 @@ static class ProcessSmapMemoryInfo {
     private int sharedDirty;
     private int privateClean;
     private int privateDirty;
+    private int anonymous;
     private int referenced;
     private String regionName;
     private String permission;
@@ -929,6 +929,10 @@ public int getReferenced() {
       return referenced;
     }
 
+    public int getAnonymous() {
+      return anonymous;
+    }
+
     public void setMemInfo(String key, String value) {
       MemInfo info = MemInfo.getMemInfoByName(key);
       int val = 0;
@@ -969,6 +973,9 @@ public void setMemInfo(String key, String value) {
       case REFERENCED:
         referenced = val;
         break;
+      case ANONYMOUS:
+        anonymous = val;
+        break;
       default:
         break;
       }
@@ -999,10 +1006,7 @@ public String toString() {
         .append(MemInfo.REFERENCED.name + ":" + this.getReferenced())
         .append(" kB\n");
       sb.append("\t")
-        .append(MemInfo.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty())
+        .append(MemInfo.ANONYMOUS.name + ":" + this.getAnonymous())
-        .append(" kB\n");
-      sb.append("\t")
-        .append(MemInfo.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty())
         .append(" kB\n");
       return sb.toString();
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java

@@ -372,18 +372,21 @@ public void createMemoryMappingInfo(ProcessTreeSmapMemInfo[] procMemInfo) {
           "7f56c177c000-7f56c177d000 "
             + "rw-p 00010000 08:02 40371558                   "
             + "/grid/0/jdk1.7.0_25/jre/lib/amd64/libnio.so",
-        new String[] { "4", "4", "25", "4", "25", "15", "10", "4", "0", "0",
+            // Format: size, rss, pss, shared_clean, shared_dirty, private_clean
+            // private_dirty, referenced, anon, anon-huge-pages, swap,
+            // kernel_page_size, mmu_page_size
+            new String[] {"4", "4", "25", "4", "25", "15", "10", "4", "10", "0",
                 "0", "4", "4"}));
       memoryMappingList.add(constructMemoryMappingInfo(
           "7fb09382e000-7fb09382f000 r--s 00003000 " + "08:02 25953545",
-        new String[] { "4", "4", "25", "4", "0", "15", "10", "4", "0", "0",
+          new String[] {"4", "4", "25", "4", "0", "15", "10", "4", "10", "0",
               "0", "4", "4"}));
       memoryMappingList.add(constructMemoryMappingInfo(
           "7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] {"4", "4",
-            "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4" }));
+              "25", "4", "0", "15", "10", "4", "10", "0", "0", "4", "4"}));
       memoryMappingList.add(constructMemoryMappingInfo(
           "7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] {"4", "4",
-            "25", "4", "50", "15", "10", "4", "0", "0", "0", "4", "4" }));
+              "25", "4", "50", "15", "10", "4", "10", "0", "0", "4", "4"}));
     }
   }
 
@@ -471,13 +474,12 @@ public void testCpuAndMemoryForProcessTree() throws IOException {
 
       // Check by enabling smaps
       setSmapsInProceTree(processTree, true);
-      // RSS=Min(shared_dirty,PSS)+PrivateClean+PrivateDirty (exclude r-xs,
+      // anon (exclude r-xs,r--s)
-      // r--s)
       Assert.assertEquals("rss memory does not match",
-        (100 * KB_TO_BYTES * 3), processTree.getRssMemorySize());
+          (20 * KB_TO_BYTES * 3), processTree.getRssMemorySize());
       // verify old API
       Assert.assertEquals("rss memory (old API) does not match",
-        (100 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem());
+          (20 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem());
 
       // test the cpu time again to see if it cumulates
       procInfos[0] =
@@ -621,10 +623,10 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
           cumuRssMem, processTree.getCumulativeRssmem());
       } else {
         Assert.assertEquals("rssmem does not include new process",
-          100 * KB_TO_BYTES * 4, processTree.getRssMemorySize());
+            20 * KB_TO_BYTES * 4, processTree.getRssMemorySize());
         // verify old API
         Assert.assertEquals("rssmem (old API) does not include new process",
-          100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem());
+            20 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem());
       }
 
       // however processes older than 1 iteration will retain the older value
@@ -650,11 +652,11 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
       } else {
         Assert.assertEquals(
           "rssmem shouldn't have included new process",
-          100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1));
+            20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1));
         // Verify old API
         Assert.assertEquals(
           "rssmem (old API) shouldn't have included new process",
-          100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1));
+            20 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1));
       }
 
       // one more process
@@ -696,11 +698,11 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
       } else {
         Assert.assertEquals(
           "rssmem shouldn't have included new processes",
-          100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2));
+            20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2));
         // Verify old API
         Assert.assertEquals(
           "rssmem (old API) shouldn't have included new processes",
-          100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2));
+            20 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2));
       }
 
       // processes older than 1 iteration should not include new process,
@@ -727,10 +729,10 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
       } else {
         Assert.assertEquals(
           "rssmem shouldn't have included new processes",
-          100 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1));
+            20 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1));
         Assert.assertEquals(
           "rssmem (old API) shouldn't have included new processes",
-          100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1));
+            20 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1));
       }
 
       // no processes older than 3 iterations