big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta

Browse Source
This commit is contained in:
Jonathan Hung 2019-09-05 14:01:42 -07:00
parent 2e2e5401f2
commit 0ccf4b0fe1
3 changed files with 63 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
View File

@ -55,6 +55,14 @@ public class CSQueueMetrics extends QueueMetrics {
MutableGaugeLong maxCapacityMB;
@Metric("Maximum CPU in virtual cores")
MutableGaugeInt maxCapacityVCores;
@Metric("Guaranteed capacity in percentage relative to parent")
private MutableGaugeFloat guaranteedCapacity;
@Metric("Guaranteed capacity in percentage relative to total partition")
private MutableGaugeFloat guaranteedAbsoluteCapacity;
@Metric("Maximum capacity in percentage relative to parent")
private MutableGaugeFloat maxCapacity;
@Metric("Maximum capacity in percentage relative to total partition")
private MutableGaugeFloat maxAbsoluteCapacity;
CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
boolean enableUserMetrics, Configuration conf) {
@ -204,4 +212,35 @@ public synchronized QueueMetrics getUserMetrics(String userName) {
return metrics;
}
public float getGuaranteedCapacity() {
return guaranteedCapacity.value();
}
public float getGuaranteedAbsoluteCapacity() {
return guaranteedAbsoluteCapacity.value();
}
public void setGuaranteedCapacities(String partition, float capacity,
float absoluteCapacity) {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
guaranteedCapacity.set(capacity);
guaranteedAbsoluteCapacity.set(absoluteCapacity);
}
}
public float getMaxCapacity() {
return maxCapacity.value();
}
public float getMaxAbsoluteCapacity() {
return maxAbsoluteCapacity.value();
}
public void setMaxCapacities(String partition, float capacity,
float absoluteCapacity) {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
maxCapacity.set(capacity);
maxAbsoluteCapacity.set(absoluteCapacity);
}
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
View File

@ -336,5 +336,11 @@ public static void updateConfiguredCapacityMetrics(ResourceCalculator rc,
queue.getMetrics().setMaxCapacityResources(partition, rc.multiplyAndNormalizeDown(
partitionResource, queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition),
queue.getMinimumAllocation()));
queue.getMetrics().setGuaranteedCapacities(partition,
queue.getQueueCapacities().getCapacity(partition),
queue.getQueueCapacities().getAbsoluteCapacity(partition));
queue.getMetrics().setMaxCapacities(partition,
queue.getQueueCapacities().getMaximumCapacity(partition),
queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition));
}
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
View File

@ -195,6 +195,8 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
private ResourceManager resourceManager = null;
private RMContext mockContext;
private static final double DELTA = 0.000001;
@Before
public void setUp() throws Exception {
resourceManager = new ResourceManager() {
@ -5502,6 +5504,22 @@ public void testCSQueueMetrics() throws Exception {
assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
.getMetrics()).getGuaranteedCapacity(), DELTA);
assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
.getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA);
assertEquals(B1_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("b1")
.getMetrics()).getGuaranteedCapacity(), DELTA);
assertEquals((B_CAPACITY / 100) * (B1_CAPACITY / 100), ((CSQueueMetrics)cs
.getQueue("b1").getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA);
assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics())
.getMaxCapacity(), DELTA);
assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics())
.getMaxAbsoluteCapacity(), DELTA);
assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics())
.getMaxCapacity(), DELTA);
assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics())
.getMaxAbsoluteCapacity(), DELTA);
// Add child queue to a, and reinitialize. Metrics should be updated
conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"} );