big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1156. Enhance NodeManager AllocatedGB and AvailableGB metrics for aggregation of decimal values. (Contributed by Tsuyoshi OZAWA)

Browse Source
This commit is contained in:
Junping Du 2014-12-03 04:11:18 -08:00
parent 7caa3bc98e
commit e65b7c5ff6
3 changed files with 29 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -112,6 +112,9 @@ Release 2.7.0 - UNRELEASED
YARN-2136. Changed RMStateStore to ignore store opearations when fenced. YARN-2136. Changed RMStateStore to ignore store opearations when fenced.
(Varun Saxena via jianhe) (Varun Saxena via jianhe)
YARN-1156. Enhance NodeManager AllocatedGB and AvailableGB metrics
for aggregation of decimal values. (Tsuyoshi OZAWA via junping_du)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
View File

@ -47,6 +47,9 @@ public class NodeManagerMetrics {
@Metric("Container launch duration") @Metric("Container launch duration")
MutableRate containerLaunchDuration; MutableRate containerLaunchDuration;
private long allocatedMB;
private long availableMB;
public static NodeManagerMetrics create() { public static NodeManagerMetrics create() {
return create(DefaultMetricsSystem.instance()); return create(DefaultMetricsSystem.instance());
} }
@ -92,22 +95,27 @@ public void endRunningContainer() {
public void allocateContainer(Resource res) { public void allocateContainer(Resource res) {
allocatedContainers.incr(); allocatedContainers.incr();
allocatedGB.incr(res.getMemory() / 1024); allocatedMB = allocatedMB + res.getMemory();
availableGB.decr(res.getMemory() / 1024); allocatedGB.set((int)Math.ceil(allocatedMB/1024d));
availableMB = availableMB - res.getMemory();
availableGB.set((int)Math.floor(availableMB/1024d));
allocatedVCores.incr(res.getVirtualCores()); allocatedVCores.incr(res.getVirtualCores());
availableVCores.decr(res.getVirtualCores()); availableVCores.decr(res.getVirtualCores());
} }
public void releaseContainer(Resource res) { public void releaseContainer(Resource res) {
allocatedContainers.decr(); allocatedContainers.decr();
allocatedGB.decr(res.getMemory() / 1024); allocatedMB = allocatedMB - res.getMemory();
availableGB.incr(res.getMemory() / 1024); allocatedGB.set((int)Math.ceil(allocatedMB/1024d));
availableMB = availableMB + res.getMemory();
availableGB.set((int)Math.floor(availableMB/1024d));
allocatedVCores.decr(res.getVirtualCores()); allocatedVCores.decr(res.getVirtualCores());
availableVCores.incr(res.getVirtualCores()); availableVCores.incr(res.getVirtualCores());
} }
public void addResource(Resource res) { public void addResource(Resource res) {
availableGB.incr(res.getMemory() / 1024); availableMB = availableMB + res.getMemory();
availableGB.incr((int)Math.floor(availableMB/1024d));
availableVCores.incr(res.getVirtualCores()); availableVCores.incr(res.getVirtualCores());
} }
@ -118,4 +126,5 @@ public void addContainerLaunchDuration(long value) {
public int getRunningContainers() { public int getRunningContainers() {
return containersRunning.value(); return containersRunning.value();
} }
} }

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
View File

@ -33,13 +33,14 @@ public class TestNodeManagerMetrics {
total.setMemory(8*GiB); total.setMemory(8*GiB);
total.setVirtualCores(16); total.setVirtualCores(16);
Resource resource = Records.newRecord(Resource.class); Resource resource = Records.newRecord(Resource.class);
resource.setMemory(1*GiB); resource.setMemory(512); //512MiB
resource.setVirtualCores(2); resource.setVirtualCores(2);
metrics.addResource(total); metrics.addResource(total);
for (int i = 5; i-- > 0;) { for (int i = 10; i-- > 0;) {
// allocate 10 containers(allocatedGB: 5GiB, availableGB: 3GiB)
metrics.launchedContainer(); metrics.launchedContainer();
metrics.allocateContainer(resource); metrics.allocateContainer(resource);
} }
@ -48,6 +49,7 @@ public class TestNodeManagerMetrics {
metrics.endInitingContainer(); metrics.endInitingContainer();
metrics.runningContainer(); metrics.runningContainer();
metrics.endRunningContainer(); metrics.endRunningContainer();
// Releasing 3 containers(allocatedGB: 3.5GiB, availableGB: 4.5GiB)
metrics.completedContainer(); metrics.completedContainer();
metrics.releaseContainer(resource); metrics.releaseContainer(resource);
@ -61,12 +63,17 @@ public class TestNodeManagerMetrics {
metrics.runningContainer(); metrics.runningContainer();
metrics.addContainerLaunchDuration(1); metrics.addContainerLaunchDuration(1);
checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6, 4, 12); // availableGB is expected to be floored,
// while allocatedGB is expected to be ceiled.
// allocatedGB: 3.5GB allocated memory is shown as 4GB
// availableGB: 4.5GB available memory is shown as 4GB
checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2);
} }
private void checkMetrics(int launched, int completed, int failed, int killed, private void checkMetrics(int launched, int completed, int failed, int killed,
int initing, int running, int allocatedGB, int initing, int running, int allocatedGB,
int allocatedContainers, int availableGB, int allocatedVCores, int availableVCores) { int allocatedContainers, int availableGB, int allocatedVCores,
int availableVCores) {
MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics"); MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics");
assertCounter("ContainersLaunched", launched, rb); assertCounter("ContainersLaunched", launched, rb);
assertCounter("ContainersCompleted", completed, rb); assertCounter("ContainersCompleted", completed, rb);