From 157af0cb225e5a6f52761533c0d24f1960f9239b Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Sun, 5 Mar 2023 21:25:16 +0530 Subject: [PATCH] YARN-11383. Workflow priority mappings is case sensitive (#5171) Contributed by Aparajita Choudhary (cherry picked from commit 2a0dc2ab2f5fb46dc540ed440d6c8b2896dd195b) --- .../WorkflowPriorityMappingsManager.java | 20 +++++++++---------- ...acitySchedulerWorkflowPriorityMapping.java | 16 +++++---------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/WorkflowPriorityMappingsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/WorkflowPriorityMappingsManager.java index fa6977aadf..b338d9af25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/WorkflowPriorityMappingsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/WorkflowPriorityMappingsManager.java @@ -56,8 +56,8 @@ public class WorkflowPriorityMappingsManager { private boolean overrideWithPriorityMappings = false; // Map of queue to a map of workflow ID to priority - private Map> priorityMappings = - new HashMap>(); + private Map> priorityMappings = + new HashMap<>(); public static class WorkflowPriorityMapping { String workflowID; @@ -115,10 +115,9 @@ public void initialize(CapacityScheduler scheduler) throws IOException { * * @return workflowID to priority mappings for a queue */ - public Map> + public Map> getWorkflowPriorityMappings() { - Map> mappings = - new HashMap>(); + Map> mappings = new HashMap<>(); Collection workflowMappings = conf.getWorkflowPriorityMappings(); for (String workflowMapping : workflowMappings) { @@ -127,9 +126,9 @@ public void initialize(CapacityScheduler scheduler) throws IOException { if (mapping != null) { if (!mappings.containsKey(mapping.queue)) { mappings.put(mapping.queue, - new HashMap()); + new HashMap()); } - mappings.get(mapping.queue).put(mapping.workflowID, mapping); + mappings.get(mapping.queue).put(mapping.workflowID, mapping.priority); } } return mappings; @@ -150,8 +149,9 @@ private WorkflowPriorityMapping getWorkflowMappingFromString( } WorkflowPriorityMapping mapping; try { - mapping = new WorkflowPriorityMapping(mappingArray[0], mappingArray[1], - Priority.newInstance(Integer.parseInt(mappingArray[2]))); + //Converting workflow id to lowercase as yarn converts application tags also to lowercase + mapping = new WorkflowPriorityMapping(StringUtils.toLowerCase(mappingArray[0]), + mappingArray[1], Priority.newInstance(Integer.parseInt(mappingArray[2]))); } catch (NumberFormatException e) { throw new IllegalArgumentException( "Illegal workflow priority for mapping " + mappingString); @@ -168,7 +168,7 @@ public Priority getMappedPriority(String workflowID, CSQueue queue) { String queuePath = queue.getQueuePath(); if (priorityMappings.containsKey(queuePath) && priorityMappings.get(queuePath).containsKey(workflowID)) { - return priorityMappings.get(queuePath).get(workflowID).priority; + return priorityMappings.get(queuePath).get(workflowID); } else { queue = queue.getParent(); return getMappedPriority(workflowID, queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java index 63fc37e997..ab48be5a2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java @@ -64,7 +64,7 @@ private static void setWorkFlowPriorityMappings( List mappings = Arrays.asList( new WorkflowPriorityMapping("workflow1", B, Priority.newInstance(2)), new WorkflowPriorityMapping("workflow2", A1, Priority.newInstance(3)), - new WorkflowPriorityMapping("workflow3", A, Priority.newInstance(4))); + new WorkflowPriorityMapping("Workflow3", A, Priority.newInstance(4))); conf.setWorkflowPriorityMappings(mappings); } @@ -85,16 +85,10 @@ public void testWorkflowPriorityMappings() throws Exception { mockRM.start(); cs.start(); - Map> expected = ImmutableMap.of( - A, ImmutableMap.of("workflow3", - new WorkflowPriorityMapping( - "workflow3", A, Priority.newInstance(4))), - B, ImmutableMap.of("workflow1", - new WorkflowPriorityMapping( - "workflow1", B, Priority.newInstance(2))), - A1, ImmutableMap.of("workflow2", - new WorkflowPriorityMapping( - "workflow2", A1, Priority.newInstance(3)))); + Map expected = ImmutableMap.of( + A, ImmutableMap.of("workflow3", Priority.newInstance(4)), + B, ImmutableMap.of("workflow1", Priority.newInstance(2)), + A1, ImmutableMap.of("workflow2", Priority.newInstance(3))); assertEquals(expected, cs.getWorkflowPriorityMappingsManager() .getWorkflowPriorityMappings());