From 24aa462673d392fed859f8088acf9679ae62a129 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Ozawa Date: Fri, 23 Jan 2015 20:37:05 +0900 Subject: [PATCH] YARN-2800. Remove MemoryNodeLabelsStore and add a way to enable/disable node labels feature. Contributed by Wangda Tan. --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/conf/YarnConfiguration.java | 16 +++-- .../TestDistributedShell.java | 1 + .../nodelabels/CommonNodeLabelsManager.java | 42 ++++++++++++- .../TestCommonNodeLabelsManager.java | 62 ++++++++++++++++++- .../TestFileSystemNodeLabelsStore.java | 1 + .../resourcemanager/ResourceManager.java | 6 +- .../yarn/server/resourcemanager/MockRM.java | 4 +- .../server/resourcemanager/TestRMRestart.java | 1 + .../nodelabels/NullRMNodeLabelsManager.java} | 10 ++- .../nodelabels/TestRMNodeLabelsManager.java | 9 ++- .../capacity/TestCapacityScheduler.java | 6 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 4 +- .../capacity/TestContainerAllocation.java | 6 +- .../scheduler/capacity/TestQueueParsing.java | 12 ++-- .../resourcemanager/webapp/TestRMWebApp.java | 6 +- 16 files changed, 152 insertions(+), 37 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/{main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/MemoryRMNodeLabelsManager.java => test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java} (85%) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7f0628da70..7263c6f071 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -197,6 +197,9 @@ Release 2.7.0 - UNRELEASED YARN-2984. Metrics for container's actual memory usage. (kasha) + YARN-2800. Remove MemoryNodeLabelsStore and add a way to enable/disable + node labels feature. (Wangda Tan via ozawa) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9ac54383a2..2ccd894929 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1648,14 +1648,10 @@ private static void addDeprecatedKeys() { public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY .name(); - public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels."; - /** - * Class for RMNodeLabelsManager Please note this value should be consistent - * in client nodes and RM node(s) + * Node-labels configurations */ - public static final String RM_NODE_LABELS_MANAGER_CLASS = NODE_LABELS_PREFIX - + "manager-class"; + public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels."; /** URI for NodeLabelManager */ public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX @@ -1664,6 +1660,14 @@ private static void addDeprecatedKeys() { NODE_LABELS_PREFIX + "fs-store.retry-policy-spec"; public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC = "2000, 500"; + + /** + * Flag to indicate if the node labels feature enabled, by default it's + * disabled + */ + public static final String NODE_LABELS_ENABLED = NODE_LABELS_PREFIX + + "enabled"; + public static final boolean DEFAULT_NODE_LABELS_ENABLED = false; public YarnConfiguration() { super(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 1d3a1040cf..46b5850ebb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -80,6 +80,7 @@ protected void setupInternal(int numNodeManager) throws Exception { conf.set("yarn.log.dir", "target"); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName()); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); if (yarnCluster == null) { yarnCluster = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index aeefff1fc6..daa1529414 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -41,6 +41,7 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -52,6 +53,7 @@ import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent; import org.apache.hadoop.yarn.util.resource.Resources; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; public class CommonNodeLabelsManager extends AbstractService { @@ -64,6 +66,14 @@ public class CommonNodeLabelsManager extends AbstractService { private static final Pattern LABEL_PATTERN = Pattern .compile("^[0-9a-zA-Z][0-9a-zA-Z-_]*"); public static final int WILDCARD_PORT = 0; + + /** + * Error messages + */ + @VisibleForTesting + public static final String NODE_LABELS_NOT_ENABLED_ERR = + "Node-label-based scheduling is disabled. Please check " + + YarnConfiguration.NODE_LABELS_ENABLED; /** * If a user doesn't specify label of a queue or node, it belongs @@ -82,6 +92,7 @@ public class CommonNodeLabelsManager extends AbstractService { protected final WriteLock writeLock; protected NodeLabelsStore store; + private boolean nodeLabelsEnabled = false; /** * A Host can have multiple Nodes @@ -193,7 +204,13 @@ protected void initDispatcher(Configuration conf) { @Override protected void serviceInit(Configuration conf) throws Exception { - initNodeLabelStore(conf); + // set if node labels enabled + nodeLabelsEnabled = + conf.getBoolean(YarnConfiguration.NODE_LABELS_ENABLED, + YarnConfiguration.DEFAULT_NODE_LABELS_ENABLED); + if (nodeLabelsEnabled) { + initNodeLabelStore(conf); + } labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL)); } @@ -250,6 +267,10 @@ protected void serviceStop() throws Exception { */ @SuppressWarnings("unchecked") public void addToCluserNodeLabels(Set labels) throws IOException { + if (!nodeLabelsEnabled) { + LOG.error(NODE_LABELS_NOT_ENABLED_ERR); + throw new IOException(NODE_LABELS_NOT_ENABLED_ERR); + } if (null == labels || labels.isEmpty()) { return; } @@ -304,6 +325,10 @@ protected void checkAddLabelsToNode( */ public void addLabelsToNode(Map> addedLabelsToNode) throws IOException { + if (!nodeLabelsEnabled) { + LOG.error(NODE_LABELS_NOT_ENABLED_ERR); + throw new IOException(NODE_LABELS_NOT_ENABLED_ERR); + } addedLabelsToNode = normalizeNodeIdToLabels(addedLabelsToNode); checkAddLabelsToNode(addedLabelsToNode); internalUpdateLabelsOnNodes(addedLabelsToNode, NodeLabelUpdateOperation.ADD); @@ -370,6 +395,11 @@ protected void internalRemoveFromClusterNodeLabels(Collection labelsToRe */ public void removeFromClusterNodeLabels(Collection labelsToRemove) throws IOException { + if (!nodeLabelsEnabled) { + LOG.error(NODE_LABELS_NOT_ENABLED_ERR); + throw new IOException(NODE_LABELS_NOT_ENABLED_ERR); + } + labelsToRemove = normalizeLabels(labelsToRemove); checkRemoveFromClusterNodeLabels(labelsToRemove); @@ -532,6 +562,11 @@ protected void internalUpdateLabelsOnNodes( public void removeLabelsFromNode(Map> removeLabelsFromNode) throws IOException { + if (!nodeLabelsEnabled) { + LOG.error(NODE_LABELS_NOT_ENABLED_ERR); + throw new IOException(NODE_LABELS_NOT_ENABLED_ERR); + } + removeLabelsFromNode = normalizeNodeIdToLabels(removeLabelsFromNode); checkRemoveLabelsFromNode(removeLabelsFromNode); @@ -567,6 +602,11 @@ protected void checkReplaceLabelsOnNode( */ public void replaceLabelsOnNode(Map> replaceLabelsToNode) throws IOException { + if (!nodeLabelsEnabled) { + LOG.error(NODE_LABELS_NOT_ENABLED_ERR); + throw new IOException(NODE_LABELS_NOT_ENABLED_ERR); + } + replaceLabelsToNode = normalizeNodeIdToLabels(replaceLabelsToNode); checkReplaceLabelsOnNode(replaceLabelsToNode); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java index c0b05e3b08..242f59caf2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java @@ -26,6 +26,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -41,7 +43,9 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { @Before public void before() { mgr = new DummyCommonNodeLabelsManager(); - mgr.init(new Configuration()); + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + mgr.init(conf); mgr.start(); } @@ -319,4 +323,58 @@ public void testReplaceLabelsOnHostsShouldUpdateNodesBelongTo() toSet("p1"), toNodeId("n1:1"), toSet("p2"), toNodeId("n1:2"), toSet("p1"))); } -} \ No newline at end of file + + private void assertNodeLabelsDisabledErrorMessage(IOException e) { + Assert.assertEquals(CommonNodeLabelsManager.NODE_LABELS_NOT_ENABLED_ERR, + e.getMessage()); + } + + @Test(timeout = 5000) + public void testNodeLabelsDisabled() throws IOException { + DummyCommonNodeLabelsManager mgr = new DummyCommonNodeLabelsManager(); + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + mgr.init(conf); + mgr.start(); + + // add labels + try { + mgr.addToCluserNodeLabels(ImmutableSet.of("x")); + } catch (IOException e) { + assertNodeLabelsDisabledErrorMessage(e); + } + + // remove labels + try { + mgr.removeFromClusterNodeLabels(ImmutableSet.of("x")); + } catch (IOException e) { + assertNodeLabelsDisabledErrorMessage(e); + } + + // add labels to node + try { + mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("host", 0), + CommonNodeLabelsManager.EMPTY_STRING_SET)); + } catch (IOException e) { + assertNodeLabelsDisabledErrorMessage(e); + } + + // remove labels from node + try { + mgr.removeLabelsFromNode(ImmutableMap.of(NodeId.newInstance("host", 0), + CommonNodeLabelsManager.EMPTY_STRING_SET)); + } catch (IOException e) { + assertNodeLabelsDisabledErrorMessage(e); + } + + // replace labels on node + try { + mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("host", 0), + CommonNodeLabelsManager.EMPTY_STRING_SET)); + } catch (IOException e) { + assertNodeLabelsDisabledErrorMessage(e); + } + + mgr.close(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 45a2d8d32f..55e0e17fa7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -63,6 +63,7 @@ private FileSystemNodeLabelsStore getStore() { public void before() throws IOException { mgr = new MockNodeLabelManager(); conf = new Configuration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); File tempDir = File.createTempFile("nlb", ".tmp"); tempDir.delete(); tempDir.mkdirs(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index cceee2baa3..4f242e93ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -334,10 +333,7 @@ protected AMLivelinessMonitor createAMLivelinessMonitor() { protected RMNodeLabelsManager createNodeLabelManager() throws InstantiationException, IllegalAccessException { - Class nlmCls = - conf.getClass(YarnConfiguration.RM_NODE_LABELS_MANAGER_CLASS, - MemoryRMNodeLabelsManager.class, RMNodeLabelsManager.class); - return nlmCls.newInstance(); + return new RMNodeLabelsManager(); } protected DelegationTokenRenewer createDelegationTokenRenewer() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 5794b435cd..06c6b3275e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -59,7 +59,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -115,7 +115,7 @@ public MockRM(Configuration conf, RMStateStore store) { @Override protected RMNodeLabelsManager createNodeLabelManager() { - RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager(); + RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index bd6fdbab46..fc419ac4c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -2073,6 +2073,7 @@ public void testRMRestartRecoveringNodeLabelManager() throws Exception { MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); MockRM rm1 = new MockRM(conf, memStore) { @Override protected RMNodeLabelsManager createNodeLabelManager() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/MemoryRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java similarity index 85% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/MemoryRMNodeLabelsManager.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java index 89053ca9ba..b1be525ae8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/MemoryRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java @@ -25,9 +25,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore; -public class MemoryRMNodeLabelsManager extends RMNodeLabelsManager { +public class NullRMNodeLabelsManager extends RMNodeLabelsManager { Map> lastNodeToLabels = null; Collection lastAddedlabels = null; Collection lastRemovedlabels = null; @@ -79,4 +80,11 @@ protected void startDispatcher() { protected void stopDispatcher() { // do nothing } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + // always enable node labels while using MemoryRMNodeLabelsManager + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + super.serviceInit(conf); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index e4cdc71c78..4b8fe76676 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.NodeLabel; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; @@ -44,12 +45,14 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { private final Resource SMALL_RESOURCE = Resource.newInstance(100, 0); private final Resource LARGE_NODE = Resource.newInstance(1000, 0); - MemoryRMNodeLabelsManager mgr = null; + NullRMNodeLabelsManager mgr = null; @Before public void before() { - mgr = new MemoryRMNodeLabelsManager(); - mgr.init(new Configuration()); + mgr = new NullRMNodeLabelsManager(); + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + mgr.init(conf); mgr.start(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 2aa57a0d79..85696a1d6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -84,7 +84,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; @@ -154,7 +154,7 @@ public void setUp() throws Exception { resourceManager = new ResourceManager() { @Override protected RMNodeLabelsManager createNodeLabelManager() { - RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager(); + RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } @@ -1485,7 +1485,7 @@ public void testMoveAppViolateQueueState() throws Exception { resourceManager = new ResourceManager() { @Override protected RMNodeLabelsManager createNodeLabelManager() { - RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager(); + RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index 261fa01307..923c0a3499 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; @@ -54,7 +54,7 @@ public void setUp() throws Exception { conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); - mgr = new MemoryRMNodeLabelsManager(); + mgr = new NullRMNodeLabelsManager(); mgr.init(conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 9a29bff970..674096efb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -45,7 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -81,7 +81,7 @@ public void setUp() throws Exception { conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); - mgr = new MemoryRMNodeLabelsManager(); + mgr = new NullRMNodeLabelsManager(); mgr.init(conf); } @@ -451,7 +451,7 @@ private Configuration getComplexConfigurationWithQueueLabels( @Test(timeout = 300000) public void testContainerAllocationWithSingleUserLimits() throws Exception { - final RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager(); + final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(conf); // set node -> label diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 5a9fbe1503..f821e64d3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -47,7 +47,7 @@ public class TestQueueParsing { @Before public void setup() { - nodeLabelManager = new MemoryRMNodeLabelsManager(); + nodeLabelManager = new NullRMNodeLabelsManager(); nodeLabelManager.init(new YarnConfiguration()); nodeLabelManager.start(); } @@ -566,7 +566,7 @@ public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager() new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); - RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager(); + RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); @@ -594,7 +594,7 @@ public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); - RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager(); + RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); @@ -622,7 +622,7 @@ public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager() new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); - RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager(); + RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); @@ -649,7 +649,7 @@ public void testQueueParsingWhenLabelsNotExist() throws IOException { new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); - RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager(); + RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index abc701db19..fb1e61de05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -179,7 +179,7 @@ public ConcurrentMap getRMNodes() { return nodesMap; } }; - rmContext.setNodeLabelManager(new MemoryRMNodeLabelsManager()); + rmContext.setNodeLabelManager(new NullRMNodeLabelsManager()); return rmContext; } @@ -211,7 +211,7 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException { null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null); - rmContext.setNodeLabelManager(new MemoryRMNodeLabelsManager()); + rmContext.setNodeLabelManager(new NullRMNodeLabelsManager()); cs.setRMContext(rmContext); cs.init(conf); return cs;