diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 48ec147aef..bc2864691f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -650,26 +650,34 @@ private void checkLimit(ContainerId containerId, String pId, if (isVmemCheckEnabled() && isProcessTreeOverLimit(containerId.toString(), currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) { + // The current usage (age=0) is always higher than the aged usage. We + // do not show the aged size in the message, base the delta on the + // current usage + long delta = currentVmemUsage - vmemLimit; // Container (the root process) is still alive and overflowing // memory. // Dump the process-tree and then clean it up. msg = formatErrorMessage("virtual", formatUsageString(currentVmemUsage, vmemLimit, currentPmemUsage, pmemLimit), - pId, containerId, pTree); + pId, containerId, pTree, delta); isMemoryOverLimit = true; containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_VMEM; } else if (isPmemCheckEnabled() && isProcessTreeOverLimit(containerId.toString(), currentPmemUsage, curRssMemUsageOfAgedProcesses, pmemLimit)) { + // The current usage (age=0) is always higher than the aged usage. We + // do not show the aged size in the message, base the delta on the + // current usage + long delta = currentPmemUsage - pmemLimit; // Container (the root process) is still alive and overflowing // memory. // Dump the process-tree and then clean it up. msg = formatErrorMessage("physical", formatUsageString(currentVmemUsage, vmemLimit, currentPmemUsage, pmemLimit), - pId, containerId, pTree); + pId, containerId, pTree, delta); isMemoryOverLimit = true; containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM; } @@ -726,11 +734,11 @@ private void reportResourceUsage(ContainerId containerId, */ private String formatErrorMessage(String memTypeExceeded, String usageString, String pId, ContainerId containerId, - ResourceCalculatorProcessTree pTree) { + ResourceCalculatorProcessTree pTree, long delta) { return String.format("Container [pid=%s,containerID=%s] is " + - "running beyond %s memory limits. ", - pId, containerId, memTypeExceeded) + + "running %dB beyond the '%S' memory limit. ", + pId, containerId, delta, memTypeExceeded) + "Current usage: " + usageString + ". Killing container.\n" + "Dump of the process-tree for " + containerId + " :\n" + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 5f72a4c25a..412b8cd502 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -332,8 +332,8 @@ public void testContainerKillOnMemoryOverflow() throws IOException, Assert.assertEquals(ContainerExitStatus.KILLED_EXCEEDED_VMEM, containerStatus.getExitStatus()); String expectedMsgPattern = - "Container \\[pid=" + pid + ",containerID=" + cId - + "\\] is running beyond virtual memory limits. Current usage: " + "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running " + + "[0-9]+B beyond the 'VIRTUAL' memory limit. Current usage: " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. " + "Killing container.\nDump of the process-tree for "