YARN-4768. getAvailablePhysicalMemorySize can be inaccurate on linux. Contributed by Nathan Roberts.
This commit is contained in:
Eric Payne
parent
025219b12f
commit
6b1c1cb01c
@ -53,7 +53,7 @@ public class SysInfoLinux extends SysInfo {
|
||||
*/
|
||||
private static final String PROCFS_MEMFILE = "/proc/meminfo";
|
||||
private static final Pattern PROCFS_MEMFILE_FORMAT =
|
||||
Pattern.compile("^([a-zA-Z]*):[ \t]*([0-9]*)[ \t]kB");
|
||||
Pattern.compile("^([a-zA-Z_()]*):[ \t]*([0-9]*)[ \t]*(kB)?");
|
||||
|
||||
// We need the values for the following keys in meminfo
|
||||
private static final String MEMTOTAL_STRING = "MemTotal";
|
||||
@ -61,6 +61,12 @@ public class SysInfoLinux extends SysInfo {
|
||||
private static final String MEMFREE_STRING = "MemFree";
|
||||
private static final String SWAPFREE_STRING = "SwapFree";
|
||||
private static final String INACTIVE_STRING = "Inactive";
|
||||
private static final String INACTIVEFILE_STRING = "Inactive(file)";
|
||||
private static final String HARDWARECORRUPTED_STRING = "HardwareCorrupted";
|
||||
private static final String HUGEPAGESTOTAL_STRING = "HugePages_Total";
|
||||
private static final String HUGEPAGESIZE_STRING = "Hugepagesize";
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Patterns for parsing /proc/cpuinfo.
|
||||
@ -122,7 +128,13 @@ public class SysInfoLinux extends SysInfo {
|
||||
private long swapSize = 0;
|
||||
private long ramSizeFree = 0; // free ram space on the machine (kB)
|
||||
private long swapSizeFree = 0; // free swap space on the machine (kB)
|
||||
private long inactiveSize = 0; // inactive cache memory (kB)
|
||||
private long inactiveSize = 0; // inactive memory (kB)
|
||||
private long inactiveFileSize = -1; // inactive cache memory, -1 if not there
|
||||
private long hardwareCorruptSize = 0; // RAM corrupt and not available
|
||||
private long hugePagesTotal = 0; // # of hugepages reserved
|
||||
private long hugePageSize = 0; // # size of each hugepage
|
||||
|
||||
|
||||
/* number of logical processors on the system. */
|
||||
private int numProcessors = 0;
|
||||
/* number of physical cores on the system. */
|
||||
@ -245,6 +257,14 @@ private void readProcMemInfoFile(boolean readAgain) {
|
||||
swapSizeFree = Long.parseLong(mat.group(2));
|
||||
} else if (mat.group(1).equals(INACTIVE_STRING)) {
|
||||
inactiveSize = Long.parseLong(mat.group(2));
|
||||
} else if (mat.group(1).equals(INACTIVEFILE_STRING)) {
|
||||
inactiveFileSize = Long.parseLong(mat.group(2));
|
||||
} else if (mat.group(1).equals(HARDWARECORRUPTED_STRING)) {
|
||||
hardwareCorruptSize = Long.parseLong(mat.group(2));
|
||||
} else if (mat.group(1).equals(HUGEPAGESTOTAL_STRING)) {
|
||||
hugePagesTotal = Long.parseLong(mat.group(2));
|
||||
} else if (mat.group(1).equals(HUGEPAGESIZE_STRING)) {
|
||||
hugePageSize = Long.parseLong(mat.group(2));
|
||||
}
|
||||
}
|
||||
str = in.readLine();
|
||||
@ -554,28 +574,31 @@ int readDiskBlockInformation(String diskName, int defSector) {
|
||||
@Override
|
||||
public long getPhysicalMemorySize() {
|
||||
readProcMemInfoFile();
|
||||
return ramSize * 1024;
|
||||
return (ramSize
|
||||
- hardwareCorruptSize
|
||||
- (hugePagesTotal * hugePageSize)) * 1024;
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public long getVirtualMemorySize() {
|
||||
readProcMemInfoFile();
|
||||
return (ramSize + swapSize) * 1024;
|
||||
return getPhysicalMemorySize() + (swapSize * 1024);
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public long getAvailablePhysicalMemorySize() {
|
||||
readProcMemInfoFile(true);
|
||||
return (ramSizeFree + inactiveSize) * 1024;
|
||||
long inactive = inactiveFileSize != -1
|
||||
? inactiveFileSize
|
||||
: inactiveSize;
|
||||
return (ramSizeFree + inactive) * 1024;
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public long getAvailableVirtualMemorySize() {
|
||||
readProcMemInfoFile(true);
|
||||
return (ramSizeFree + swapSizeFree + inactiveSize) * 1024;
|
||||
return getAvailablePhysicalMemorySize() + (swapSizeFree * 1024);
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
|
||||
@ -29,6 +29,7 @@
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
||||
/**
|
||||
* A JUnit test to test {@link SysInfoLinux}
|
||||
@ -110,11 +111,56 @@ int readDiskBlockInformation(String diskName, int defSector) {
|
||||
"VmallocTotal: 34359738367 kB\n" +
|
||||
"VmallocUsed: 1632 kB\n" +
|
||||
"VmallocChunk: 34359736375 kB\n" +
|
||||
"HugePages_Total: 0\n" +
|
||||
"HugePages_Total: %d\n" +
|
||||
"HugePages_Free: 0\n" +
|
||||
"HugePages_Rsvd: 0\n" +
|
||||
"Hugepagesize: 2048 kB";
|
||||
|
||||
static final String MEMINFO_FORMAT_2 =
|
||||
"MemTotal: %d kB\n" +
|
||||
"MemFree: %d kB\n" +
|
||||
"Buffers: 129976 kB\n" +
|
||||
"Cached: 32317676 kB\n" +
|
||||
"SwapCached: 0 kB\n" +
|
||||
"Active: 88938588 kB\n" +
|
||||
"Inactive: %d kB\n" +
|
||||
"Active(anon): 77502200 kB\n" +
|
||||
"Inactive(anon): 6385336 kB\n" +
|
||||
"Active(file): 11436388 kB\n" +
|
||||
"Inactive(file): %d kB\n" +
|
||||
"Unevictable: 0 kB\n" +
|
||||
"Mlocked: 0 kB\n" +
|
||||
"SwapTotal: %d kB\n" +
|
||||
"SwapFree: %d kB\n" +
|
||||
"Dirty: 575864 kB\n" +
|
||||
"Writeback: 16 kB\n" +
|
||||
"AnonPages: 83886180 kB\n" +
|
||||
"Mapped: 108640 kB\n" +
|
||||
"Shmem: 1880 kB\n" +
|
||||
"Slab: 2413448 kB\n" +
|
||||
"SReclaimable: 2194488 kB\n" +
|
||||
"SUnreclaim: 218960 kB\n" +
|
||||
"KernelStack: 31496 kB\n" +
|
||||
"PageTables: 195176 kB\n" +
|
||||
"NFS_Unstable: 0 kB\n" +
|
||||
"Bounce: 0 kB\n" +
|
||||
"WritebackTmp: 0 kB\n" +
|
||||
"CommitLimit: 97683468 kB\n" +
|
||||
"Committed_AS: 94553560 kB\n" +
|
||||
"VmallocTotal: 34359738367 kB\n" +
|
||||
"VmallocUsed: 498580 kB\n" +
|
||||
"VmallocChunk: 34256922296 kB\n" +
|
||||
"HardwareCorrupted: %d kB\n" +
|
||||
"AnonHugePages: 0 kB\n" +
|
||||
"HugePages_Total: %d\n" +
|
||||
"HugePages_Free: 0\n" +
|
||||
"HugePages_Rsvd: 0\n" +
|
||||
"HugePages_Surp: 0\n" +
|
||||
"Hugepagesize: 2048 kB\n" +
|
||||
"DirectMap4k: 4096 kB\n" +
|
||||
"DirectMap2M: 2027520 kB\n" +
|
||||
"DirectMap1G: 132120576 kB\n";
|
||||
|
||||
static final String CPUINFO_FORMAT =
|
||||
"processor : %s\n" +
|
||||
"vendor_id : AuthenticAMD\n" +
|
||||
@ -285,19 +331,57 @@ public void parsingProcMemFile() throws IOException {
|
||||
long inactive = 567732L;
|
||||
long swapTotal = 2096472L;
|
||||
long swapFree = 1818480L;
|
||||
int nrHugePages = 10;
|
||||
File tempFile = new File(FAKE_MEMFILE);
|
||||
tempFile.deleteOnExit();
|
||||
FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
|
||||
fWriter.write(String.format(MEMINFO_FORMAT,
|
||||
memTotal, memFree, inactive, swapTotal, swapFree));
|
||||
memTotal, memFree, inactive, swapTotal, swapFree, nrHugePages));
|
||||
|
||||
fWriter.close();
|
||||
assertEquals(plugin.getAvailablePhysicalMemorySize(),
|
||||
1024L * (memFree + inactive));
|
||||
assertEquals(plugin.getAvailableVirtualMemorySize(),
|
||||
1024L * (memFree + inactive + swapFree));
|
||||
assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal);
|
||||
assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal));
|
||||
assertEquals(plugin.getPhysicalMemorySize(),
|
||||
1024L * (memTotal - (nrHugePages * 2048)));
|
||||
assertEquals(plugin.getVirtualMemorySize(),
|
||||
1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test parsing /proc/meminfo with Inactive(file) present
|
||||
* @throws IOException
|
||||
*/
|
||||
@Test
|
||||
public void parsingProcMemFile2() throws IOException {
|
||||
long memTotal = 131403836L;
|
||||
long memFree = 11257036L;
|
||||
long inactive = 27396032L;
|
||||
long inactiveFile = 21010696L;
|
||||
long swapTotal = 31981552L;
|
||||
long swapFree = 1818480L;
|
||||
long hardwareCorrupt = 31960904L;
|
||||
int nrHugePages = 10;
|
||||
File tempFile = new File(FAKE_MEMFILE);
|
||||
tempFile.deleteOnExit();
|
||||
FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
|
||||
fWriter.write(String.format(MEMINFO_FORMAT_2,
|
||||
memTotal, memFree, inactive, inactiveFile, swapTotal, swapFree,
|
||||
hardwareCorrupt, nrHugePages));
|
||||
|
||||
fWriter.close();
|
||||
assertEquals(plugin.getAvailablePhysicalMemorySize(),
|
||||
1024L * (memFree + inactiveFile));
|
||||
assertFalse(plugin.getAvailablePhysicalMemorySize() ==
|
||||
1024L * (memFree + inactive));
|
||||
assertEquals(plugin.getAvailableVirtualMemorySize(),
|
||||
1024L * (memFree + inactiveFile + swapFree));
|
||||
assertEquals(plugin.getPhysicalMemorySize(),
|
||||
1024L * (memTotal - hardwareCorrupt - (nrHugePages * 2048)));
|
||||
assertEquals(plugin.getVirtualMemorySize(),
|
||||
1024L * (memTotal - hardwareCorrupt -
|
||||
(nrHugePages * 2048) + swapTotal));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Loading…
Reference in New Issue
Block a user