big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11016. Queue weight is incorrectly reset to zero. Contributed by Andras Gyori

Browse Source
This commit is contained in:
Szilard Nemeth 2021-12-07 15:55:16 +01:00
parent 7284d23476
commit 126079612c
2 changed files with 47 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
View File

@ -312,7 +312,7 @@ public void clearConfigurableFields() {
_set(label, CapacityType.MAX_CAP, 0);
_set(label, CapacityType.ABS_CAP, 0);
_set(label, CapacityType.ABS_MAX_CAP, 0);
_set(label, CapacityType.WEIGHT, 0);
_set(label, CapacityType.WEIGHT, -1);
}
} finally {
writeLock.unlock();

46
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
View File

@ -26,6 +26,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@ -38,6 +39,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@ -74,6 +76,28 @@ public static <E> Set<E> toSet(E... elements) {
return set;
}
public static CapacitySchedulerConfiguration getConfigWithInheritedAccessibleNodeLabel(
Configuration config) {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(
config);
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT,
new String[] { "a"});
conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 100f);
conf.setCapacityByLabel(A, "newLabel", 100f);
conf.setAccessibleNodeLabels(A, toSet("newLabel"));
conf.setAllowZeroCapacitySum(A, true);
// Define 2nd-level queues
conf.setQueues(A, new String[] { "a1" });
conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100f);
return conf;
}
/*
* Queue structure:
* root (*)
@ -368,6 +392,28 @@ public void testChildAccessibleNodeLabelsWeightMode() throws Exception {
rm.close();
}
/**
* Tests whether weight is correctly reset to -1. See YARN-11016 for further details.
* @throws IOException if reinitialization fails
*/
@Test()
public void testAccessibleNodeLabelsInheritanceNoWeightMode() throws IOException {
CapacitySchedulerConfiguration newConf = getConfigWithInheritedAccessibleNodeLabel(conf);
MockRM rm = new MockRM(newConf);
CapacityScheduler cs =
(CapacityScheduler) rm.getRMContext().getScheduler();
Resource clusterResource = Resource.newInstance(1024, 2);
cs.getRootQueue().updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
try {
cs.reinitialize(newConf, rm.getRMContext());
} catch (Exception e) {
Assert.fail("Reinitialization failed with " + e);
}
}
@Test
public void testQueueInfoWeight() throws Exception {
MockRM rm = new MockRM(conf);