big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10714. Remove dangling dynamic queues on reinitialization. Contributed by Andras Gyori

Browse Source
This commit is contained in:
Szilard Nemeth 2021-04-07 11:52:21 +02:00
parent dac60b8282
commit 9cd69c20c4
2 changed files with 97 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
View File

@ -27,8 +27,6 @@
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@ -340,11 +338,14 @@ private void updateQueues(CSQueueStore existingQueues,
}
for (CSQueue queue : existingQueues.getQueues()) {
if (!((AbstractCSQueue) queue).isDynamicQueue() && newQueues.get(
queue.getQueuePath()) == null && !(
queue instanceof AutoCreatedLeafQueue && conf
.isAutoCreateChildQueueEnabled(
queue.getParent().getQueuePath()))) {
boolean isDanglingDynamicQueue = isDanglingDynamicQueue(
newQueues, existingQueues, queue);
boolean isRemovable = isDanglingDynamicQueue || !isDynamicQueue(queue)
&& newQueues.get(queue.getQueuePath()) == null
&& !(queue instanceof AutoCreatedLeafQueue &&
conf.isAutoCreateChildQueueEnabled(queue.getParent().getQueuePath()));
if (isRemovable) {
existingQueues.remove(queue);
}
}
@ -435,4 +436,32 @@ private Map<String, Set<String>> getQueueToLabels() {
getQueueStateManager() {
return this.queueStateManager;
}
private boolean isDynamicQueue(CSQueue queue) {
return (queue instanceof AbstractCSQueue) &&
((AbstractCSQueue) queue).isDynamicQueue();
}
private boolean isDanglingDynamicQueue(
CSQueueStore newQueues, CSQueueStore existingQueues,
CSQueue queue) {
if (!isDynamicQueue(queue)) {
return false;
}
if (queue.getParent() == null) {
return true;
}
if (newQueues.get(queue.getParent().getQueuePath()) != null) {
return false;
}
CSQueue parent = existingQueues.get(queue.getParent().getQueuePath());
if (parent == null) {
return true;
}
// A dynamic queue is dangling, if its parent is not parsed in newQueues
// or if its parent is not a dynamic queue. Dynamic queues are not parsed in
// newQueues but they are deleted automatically, so it is safe to assume
// that existingQueues contain valid dynamic queues.
return !isDynamicQueue(parent);
}
}

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@ -968,6 +969,66 @@ public void testQueueInfoIfAmbiguousQueueNames() throws Exception {
bAutoLeafQueue.getQueueInfo().getQueueName());
}
@Test
public void testRemoveDanglingAutoCreatedQueuesOnReinit() throws Exception {
startScheduler();
// Validate static parent deletion
createQueue("root.a.a-auto");
AbstractCSQueue aAuto = (AbstractCSQueue) cs.
getQueue("root.a.a-auto");
Assert.assertTrue(aAuto.isDynamicQueue());
csConf.setState("root.a", QueueState.STOPPED);
cs.reinitialize(csConf, mockRM.getRMContext());
aAuto = (AbstractCSQueue) cs.
getQueue("root.a.a-auto");
Assert.assertEquals("root.a.a-auto is not in STOPPED state", QueueState.STOPPED, aAuto.getState());
csConf.setQueues("root", new String[]{"b"});
cs.reinitialize(csConf, mockRM.getRMContext());
CSQueue aAutoNew = cs.getQueue("root.a.a-auto");
Assert.assertNull(aAutoNew);
submitApp(cs, USER0, "a-auto", "root.a");
aAutoNew = cs.getQueue("root.a.a-auto");
Assert.assertNotNull(aAutoNew);
// Validate static grandparent deletion
csConf.setQueues("root", new String[]{"a", "b"});
csConf.setQueues("root.a", new String[]{"a1"});
csConf.setAutoQueueCreationV2Enabled("root.a.a1", true);
cs.reinitialize(csConf, mockRM.getRMContext());
createQueue("root.a.a1.a1-auto");
CSQueue a1Auto = cs.getQueue("root.a.a1.a1-auto");
Assert.assertNotNull("a1-auto should exist", a1Auto);
csConf.setQueues("root", new String[]{"b"});
cs.reinitialize(csConf, mockRM.getRMContext());
a1Auto = cs.getQueue("root.a.a1.a1-auto");
Assert.assertNull("a1-auto has no parent and should not exist", a1Auto);
// Validate dynamic parent deletion
csConf.setState("root.b", QueueState.STOPPED);
cs.reinitialize(csConf, mockRM.getRMContext());
csConf.setAutoQueueCreationV2Enabled("root.b", true);
cs.reinitialize(csConf, mockRM.getRMContext());
createQueue("root.b.b-auto-parent.b-auto-leaf");
CSQueue bAutoParent = cs.getQueue("root.b.b-auto-parent");
Assert.assertNotNull("b-auto-parent should exist", bAutoParent);
ParentQueue b = (ParentQueue) cs.getQueue("root.b");
b.removeChildQueue(bAutoParent);
cs.reinitialize(csConf, mockRM.getRMContext());
bAutoParent = cs.getQueue("root.b.b-auto-parent");
Assert.assertNull("b-auto-parent should not exist ", bAutoParent);
CSQueue bAutoLeaf = cs.getQueue("root.b.b-auto-parent.b-auto-leaf");
Assert.assertNull("b-auto-leaf should not exist " +
"when its dynamic parent is removed", bAutoLeaf);
}
protected LeafQueue createQueue(String queuePath) throws YarnException {
return autoQueueHandler.autoCreateQueue(
CSQueueUtils.extractQueuePath(queuePath));