From e3e9369c1d7c3045f910a7c320970b643a73c862 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Thu, 26 May 2022 07:51:55 +0100 Subject: [PATCH] YARN-11128. Fix comments in TestProportionalCapacityPreemptionPolicy* (#4271) Co-authored-by: Ashutosh Gupta Reviewed-by: Hemanth Boyina Signed-off-by: Akira Ajisaka --- ...cityPreemptionPolicyInterQueueWithDRF.java | 11 ++++--- ...nalCapacityPreemptionPolicyIntraQueue.java | 11 ++++--- ...reemptionPolicyIntraQueueFairOrdering.java | 10 +++--- ...tyPreemptionPolicyIntraQueueUserLimit.java | 33 ++++++++++--------- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java index 2720802018..efd2ffc991 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java @@ -18,6 +18,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; + import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; @@ -26,10 +31,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceUtils; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; @@ -210,7 +211,7 @@ public void testInterQueuePreemptionWithStrictAndRelaxedDRF() "-b(=[20480:200 61440:600 0:0 20480:20 0]);" + // a "-c(=[20480:200 61440:600 0:0 0:0 0])"; // c String appsConfig = - //queueName\t(priority,resource,host,expression,#repeat,reserved) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" + "(1,1024:1,n1,,60,false,0:0,user1);" + // app1 in a "b\t" + "(1,0:0,n1,,0,false,20480:20,user2);"; // app2 in b diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java index 8077962427..2fb69f5557 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java @@ -18,13 +18,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; -import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor; -import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import java.io.IOException; + import org.junit.Before; import org.junit.Test; -import java.io.IOException; +import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor; +import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; @@ -845,7 +846,7 @@ public void testIntraQueuePreemptionWithTwoUsers() "-b(=[40 100 40 120 0])"; // b String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user(optional)) "a\t" // app1 in a + "(1,1,n1,,5,false,25);" + // app1 a "a\t" // app2 in a diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueFairOrdering.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueFairOrdering.java index eb9d21836d..def4554ec5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueFairOrdering.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueFairOrdering.java @@ -74,7 +74,7 @@ public void testIntraQueuePreemptionFairOrderingPolicyEnabledOneAppPerUser() // With FairOrderingPolicy enabled on queue a, all 20 resources should be // preempted from app1 String appsConfig = - // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1, user1 in a + "(1,1,n1,,60,false,0,user1);" + "a\t" // app2, user2 in a @@ -122,7 +122,7 @@ public void testIntraQueuePreemptionFifoOrderingPolicyEnabled() // With FifoOrderingPolicy enabled on queue a, the first 5 should come from // the youngest app, app2, until app2 is reduced to the user limit of 35. String appsConfig = - // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1, user1 in a + "(1,1,n1,,60,false,0,user1);" + "a\t" // app2, user2 in a @@ -145,7 +145,7 @@ public void testIntraQueuePreemptionFifoOrderingPolicyEnabled() // The next 15 should come from app1 even though app2 is younger since app2 // has already been reduced to its user limit. appsConfig = - // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1, user1 in a + "(1,1,n1,,60,false,0,user1);" + "a\t" // app2, user2 in a @@ -195,7 +195,7 @@ public void testIntraQueuePreemptionFairOrderingPolicyMulitipleAppsPerUser() // preempted from app1 since it's the most over served app from the most // over served user String appsConfig = - // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 and app2, user1 in a + "(1,1,n1,,35,false,0,user1);" + "a\t" @@ -243,7 +243,7 @@ public void testIntraQueuePreemptionFifoOrderingPolicyMultipleAppsPerUser() // user4/app4 is requesting 20 resources in queue a // With 3 users, preemptable user limit should be around 35 resources each. String appsConfig = - // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1, user1 in a + "(1,1,n1,,40,false,0,user1);" + "a\t" // app2, user1 in a diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java index 9fb08cd3bc..2a0838b83f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java @@ -18,12 +18,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; -import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import java.io.IOException; + import org.junit.Before; import org.junit.Test; -import java.io.IOException; +import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.times; @@ -84,7 +85,7 @@ public void testSimpleIntraQueuePreemptionWithTwoUsers() "-a(=[100 100 100 30 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,1,n1,,100,false,0,user1);" + // app1 a "a\t" // app2 in a @@ -139,7 +140,7 @@ public void testNoIntraQueuePreemptionWithSingleUser() "-a(=[100 100 100 30 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,1,n1,,100,false,0,user1);" + // app1 a "a\t" // app2 in a @@ -196,7 +197,7 @@ public void testNoIntraQueuePreemptionWithTwoUserUnderUserLimit() "-a(=[100 100 80 30 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,1,n1,,50,false,0,user1);" + // app1 a "a\t" // app2 in a @@ -252,7 +253,7 @@ public void testSimpleIntraQueuePreemptionWithTwoUsersWithAppPriority() "-a(=[100 100 100 30 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(2,1,n1,,100,false,0,user1);" + // app1 a "a\t" // app2 in a @@ -309,7 +310,7 @@ public void testIntraQueuePreemptionOfUserLimitWithMultipleApps() "-a(=[100 100 80 90 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,1,n1,,30,false,30,user1);" + // app1 a "a\t" // app2 in a @@ -370,7 +371,7 @@ public void testNoPreemptionOfUserLimitWithMultipleAppsAndSameUser() "-a(=[100 100 80 90 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,1,n1,,30,false,20,user1);" + // app1 a "a\t" // app2 in a @@ -439,7 +440,7 @@ public void testIntraQueuePreemptionOfUserLimitWitAppsOfDifferentPriority() "-a(=[100 100 80 60 0])"; // b String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(3,1,n1,,30,false,30,user1);" + // app1 a "a\t" // app2 in a @@ -490,7 +491,7 @@ public void testIntraQueuePreemptionOfUserLimitInTwoQueues() "-b(=[40 100 35 20 0])"; // b String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(3,1,n1,,20,false,30,user1);" + // app1 a "a\t" // app2 in a @@ -556,7 +557,7 @@ public void testIntraQueuePreemptionWithTwoRequestingUsers() "-a(=[100 100 100 20 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,1,n1,,60,false,10,user1);" + // app1 a "a\t" // app2 in a @@ -795,7 +796,7 @@ public void testNoIntraQueuePreemptionWithUserLimitDeadzone() "-a(=[100 100 100 20 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,3,n1,,20,false,10,user1);" + // app1 a "a\t" // app2 in a @@ -855,7 +856,7 @@ public void testIntraQueuePreemptionWithUserLimitDeadzoneAndPriority() "-a(=[100 100 100 20 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,3,n1,,20,false,10,user1);" + // app1 a "a\t" // app2 in a @@ -879,7 +880,7 @@ public void testIntraQueuePreemptionWithUserLimitDeadzoneAndPriority() // After first round, 3 containers were preempted from app1 and resource // distribution will be like below. appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,3,n1,,17,false,10,user1);" + // app1 a "a\t" // app2 in a @@ -914,7 +915,7 @@ public void testSimpleIntraQueuePreemptionOneUserUnderOneUserAtOneUserAbove() "-a(=[100 100 100 1 0])"; // a String appsConfig = - // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) + // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user) "a\t" // app1 in a + "(1,1,n1,,65,false,0,user1);" + "a\t" // app2 in a