big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9017. PlacementRule order is not maintained in CS. Contributed by Bilwa S T.

Browse Source 
(cherry picked from commit 35010120fb)
This commit is contained in:
Inigo Goiri 2020-05-06 13:22:54 -07:00 committed by Brahma Reddy Battula
parent 97171b9b18
commit 8c8ef2f444
3 changed files with 50 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
View File

@ -703,8 +703,11 @@ public void updatePlacementRules() throws IOException {
Set<String> distinguishRuleSet = CapacitySchedulerConfigValidator
.validatePlacementRules(placementRuleStrs);
// add UserGroupMappingPlacementRule if absent
distinguishRuleSet.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
// add UserGroupMappingPlacementRule if empty,default value of
// yarn.scheduler.queue-placement-rules is user-group
if (distinguishRuleSet.isEmpty()) {
distinguishRuleSet.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
}
placementRuleStrs = new ArrayList<>(distinguishRuleSet);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
View File

@ -28,7 +28,7 @@
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Set;
public final class CapacitySchedulerConfigValidator {
@ -58,7 +58,7 @@ public static boolean validateCSConfiguration(
public static Set<String> validatePlacementRules(
Collection<String> placementRuleStrs) throws IOException {
Set<String> distinguishRuleSet = new HashSet<>();
Set<String> distinguishRuleSet = new LinkedHashSet<>();
// fail the case if we get duplicate placementRule add in
for (String pls : placementRuleStrs) {
if (!distinguishRuleSet.add(pls)) {

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.placement;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@ -28,7 +27,9 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
@ -49,20 +50,22 @@ public class TestPlacementManager {
public static final String PARENT_QUEUE = "c";
private MockRM mockRM = null;
private static final long CLUSTER_TIMESTAMP = System.currentTimeMillis();
private CapacitySchedulerConfiguration conf;
private String getQueueMapping(String parentQueue, String leafQueue) {
return parentQueue + DOT + leafQueue;
}
@Test
public void testPlaceApplicationWithPlacementRuleChain() throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
@Before
public void setup() {
conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
}
@Test
public void testPlaceApplicationWithPlacementRuleChain() throws Exception {
mockRM = new MockRM(conf);
CapacityScheduler cs = (CapacityScheduler) mockRM.getResourceScheduler();
mockRM.start();
@ -112,4 +115,38 @@ public void testPlaceApplicationWithPlacementRuleChain() throws Exception {
}
}
@Test
public void testPlacementRuleUpdationOrder() throws Exception {
List<QueueMapping> queueMappings = new ArrayList<>();
QueueMapping userQueueMapping = QueueMappingBuilder.create()
.type(MappingType.USER).source(USER1)
.queue(getQueueMapping(PARENT_QUEUE, USER1)).build();
UserGroupMappingPlacementRule ugRule = new UserGroupMappingPlacementRule(
false, Arrays.asList(userQueueMapping), null);
// Configure placement rule
conf.set(YarnConfiguration.QUEUE_PLACEMENT_RULES, ugRule.getName());
queueMappings.add(userQueueMapping);
conf.setQueueMappings(queueMappings);
mockRM = new MockRM(conf);
CapacityScheduler cs = (CapacityScheduler) mockRM.getResourceScheduler();
mockRM.start();
PlacementManager pm = cs.getRMContext().getQueuePlacementManager();
// As we are setting placement rule, It shouldn't update default
// placement rule ie user-group. Number of placemnt rules should be 1.
Assert.assertEquals(1, pm.getPlacementRules().size());
// Verifying if placement rule set is same as the one we configured
Assert.assertEquals(ugRule.getName(),
pm.getPlacementRules().get(0).getName());
}
@After
public void tearDown() {
if (null != mockRM) {
mockRM.stop();
}
}
}