diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index 6ae1e0206c..5540d737cb 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -191,7 +191,7 @@ public boolean isRunning() { @VisibleForTesting @SuppressFBWarnings(value="NN_NAKED_NOTIFY", justification="Used only for testing") - synchronized void processContainersNow() { + public synchronized void processContainersNow() { notify(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java index b337f4e995..18ec2c385b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java @@ -43,7 +43,8 @@ private ContainerPlacementPolicyFactory() { public static ContainerPlacementPolicy getPolicy(Configuration conf, final NodeManager nodeManager, NetworkTopology clusterMap, - final boolean fallback) throws SCMException{ + final boolean fallback, SCMContainerPlacementMetrics metrics) + throws SCMException{ final Class placementClass = conf .getClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT, @@ -51,7 +52,8 @@ public static ContainerPlacementPolicy getPolicy(Configuration conf, Constructor constructor; try { constructor = placementClass.getDeclaredConstructor(NodeManager.class, - Configuration.class, NetworkTopology.class, boolean.class); + Configuration.class, NetworkTopology.class, boolean.class, + SCMContainerPlacementMetrics.class); LOG.info("Create container placement policy of type " + placementClass.getCanonicalName()); } catch (NoSuchMethodException e) { @@ -64,7 +66,8 @@ public static ContainerPlacementPolicy getPolicy(Configuration conf, } try { - return constructor.newInstance(nodeManager, conf, clusterMap, fallback); + return constructor.newInstance(nodeManager, conf, clusterMap, fallback, + metrics); } catch (Exception e) { throw new RuntimeException("Failed to instantiate class " + placementClass.getCanonicalName() + " for " + e.getMessage()); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java index f2892ffb4c..85d281cf6d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java @@ -79,7 +79,7 @@ public final class SCMContainerPlacementCapacity extends SCMCommonPolicy { */ public SCMContainerPlacementCapacity(final NodeManager nodeManager, final Configuration conf, final NetworkTopology networkTopology, - final boolean fallback) { + final boolean fallback, final SCMContainerPlacementMetrics metrics) { super(nodeManager, conf); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementMetrics.java new file mode 100644 index 0000000000..869a65b7d5 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementMetrics.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.container.placement.algorithms; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; + +/** + * This class is for maintaining Topology aware container placement statistics. + */ +@Metrics(about="SCM Container Placement Metrics", context = "ozone") +public class SCMContainerPlacementMetrics implements MetricsSource { + public static final String SOURCE_NAME = + SCMContainerPlacementMetrics.class.getSimpleName(); + private static final MetricsInfo RECORD_INFO = Interns.info(SOURCE_NAME, + "SCM Placement Metrics"); + private static MetricsRegistry registry; + + // total datanode allocation request count + @Metric private MutableCounterLong datanodeRequestCount; + // datanode allocation attempt count, including success, fallback and failed + @Metric private MutableCounterLong datanodeChooseAttemptCount; + // datanode successful allocation count + @Metric private MutableCounterLong datanodeChooseSuccessCount; + // datanode allocated with some allocation constrains compromised + @Metric private MutableCounterLong datanodeChooseFallbackCount; + + public SCMContainerPlacementMetrics() { + } + + public static SCMContainerPlacementMetrics create() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + registry = new MetricsRegistry(RECORD_INFO); + return ms.register(SOURCE_NAME, "SCM Placement Metrics", + new SCMContainerPlacementMetrics()); + } + + public void incrDatanodeRequestCount(long count) { + System.out.println("request + 1"); + this.datanodeRequestCount.incr(count); + } + + public void incrDatanodeChooseSuccessCount() { + System.out.println("success + 1"); + this.datanodeChooseSuccessCount.incr(1); + } + + public void incrDatanodeChooseFallbackCount() { + System.out.println("fallback + 1"); + this.datanodeChooseFallbackCount.incr(1); + } + + public void incrDatanodeChooseAttemptCount() { + System.out.println("attempt + 1"); + this.datanodeChooseAttemptCount.incr(1); + } + + public void unRegister() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + ms.unregisterSource(SOURCE_NAME); + } + + @VisibleForTesting + public long getDatanodeRequestCount() { + return this.datanodeRequestCount.value(); + } + + @VisibleForTesting + public long getDatanodeChooseSuccessCount() { + return this.datanodeChooseSuccessCount.value(); + } + + @VisibleForTesting + public long getDatanodeChooseFallbackCount() { + return this.datanodeChooseFallbackCount.value(); + } + + @VisibleForTesting + public long getDatanodeChooseAttemptCount() { + return this.datanodeChooseAttemptCount.value(); + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + registry.snapshot(collector.addRecord(registry.info().name()), true); + } +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java index 14d1ffb58c..52db8a48f8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java @@ -53,6 +53,7 @@ public final class SCMContainerPlacementRackAware extends SCMCommonPolicy { private boolean fallback; private static final int RACK_LEVEL = 1; private static final int MAX_RETRY= 3; + private final SCMContainerPlacementMetrics metrics; /** * Constructs a Container Placement with rack awareness. @@ -66,10 +67,11 @@ public final class SCMContainerPlacementRackAware extends SCMCommonPolicy { */ public SCMContainerPlacementRackAware(final NodeManager nodeManager, final Configuration conf, final NetworkTopology networkTopology, - final boolean fallback) { + final boolean fallback, final SCMContainerPlacementMetrics metrics) { super(nodeManager, conf); this.networkTopology = networkTopology; this.fallback = fallback; + this.metrics = metrics; } /** @@ -93,7 +95,7 @@ public List chooseDatanodes( List excludedNodes, List favoredNodes, int nodesRequired, final long sizeRequired) throws SCMException { Preconditions.checkArgument(nodesRequired > 0); - + metrics.incrDatanodeRequestCount(nodesRequired); int datanodeCount = networkTopology.getNumOfLeafNode(NetConstants.ROOT); int excludedNodesCount = excludedNodes == null ? 0 : excludedNodes.size(); if (datanodeCount < nodesRequired + excludedNodesCount) { @@ -241,9 +243,11 @@ private Node chooseNode(List excludedNodes, Node affinityNode, int ancestorGen = RACK_LEVEL; int maxRetry = MAX_RETRY; List excludedNodesForCapacity = null; + boolean isFallbacked = false; while(true) { Node node = networkTopology.chooseRandom(NetConstants.ROOT, null, excludedNodes, affinityNode, ancestorGen); + metrics.incrDatanodeChooseAttemptCount(); if (node == null) { // cannot find the node which meets all constrains LOG.warn("Failed to find the datanode. excludedNodes:" + @@ -251,6 +255,7 @@ private Node chooseNode(List excludedNodes, Node affinityNode, ", affinityNode:" + (affinityNode == null ? "" : affinityNode.getNetworkFullPath())); if (fallback) { + isFallbacked = true; // fallback, don't consider the affinity node if (affinityNode != null) { affinityNode = null; @@ -267,11 +272,15 @@ private Node chooseNode(List excludedNodes, Node affinityNode, " excludedNodes and affinityNode constrains.", null); } if (hasEnoughSpace((DatanodeDetails)node, sizeRequired)) { - LOG.debug("Datanode {} is chosen. Required size is {}", + LOG.warn("Datanode {} is chosen. Required size is {}", node.toString(), sizeRequired); if (excludedNodes != null && excludedNodesForCapacity != null) { excludedNodes.removeAll(excludedNodesForCapacity); } + metrics.incrDatanodeChooseSuccessCount(); + if (isFallbacked) { + metrics.incrDatanodeChooseFallbackCount(); + } return node; } else { maxRetry--; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java index bcd7986ce5..6b1a5c8c6c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java @@ -51,7 +51,7 @@ public final class SCMContainerPlacementRandom extends SCMCommonPolicy */ public SCMContainerPlacementRandom(final NodeManager nodeManager, final Configuration conf, final NetworkTopology networkTopology, - final boolean fallback) { + final boolean fallback, final SCMContainerPlacementMetrics metrics) { super(nodeManager, conf); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index fdbcbd682b..f383106330 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -42,6 +42,8 @@ import org.apache.hadoop.hdds.scm.block.PendingDeleteHandler; import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicyFactory; +import org.apache.hadoop.hdds.scm.container.placement.algorithms + .SCMContainerPlacementMetrics; import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.safemode.SafeModeHandler; @@ -387,9 +389,11 @@ private void initializeSystemManagers(OzoneConfiguration conf, conf, scmStorageConfig, eventQueue, clusterMap); } + SCMContainerPlacementMetrics placementMetrics = + SCMContainerPlacementMetrics.create(); ContainerPlacementPolicy containerPlacementPolicy = ContainerPlacementPolicyFactory.getPolicy(conf, scmNodeManager, - clusterMap, true); + clusterMap, true, placementMetrics); if (configurator.getPipelineManager() != null) { pipelineManager = configurator.getPipelineManager(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java index 456b44bbc5..18c4a64a04 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java @@ -100,7 +100,8 @@ public void testRackAwarePolicy() throws IOException { .thenReturn(new SCMNodeMetric(storageCapacity, 70L, 30L)); ContainerPlacementPolicy policy = ContainerPlacementPolicyFactory - .getPolicy(conf, nodeManager, cluster, true); + .getPolicy(conf, nodeManager, cluster, true, + SCMContainerPlacementMetrics.create()); int nodeNum = 3; List datanodeDetails = @@ -117,7 +118,7 @@ public void testRackAwarePolicy() throws IOException { @Test public void testDefaultPolicy() throws IOException { ContainerPlacementPolicy policy = ContainerPlacementPolicyFactory - .getPolicy(conf, null, null, true); + .getPolicy(conf, null, null, true, null); Assert.assertSame(SCMContainerPlacementRandom.class, policy.getClass()); } @@ -138,7 +139,7 @@ public void testConstuctorNotFound() throws SCMException { // set a placement class which does't have the right constructor implemented conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, DummyImpl.class.getName()); - ContainerPlacementPolicyFactory.getPolicy(conf, null, null, true); + ContainerPlacementPolicyFactory.getPolicy(conf, null, null, true, null); } @Test(expected = RuntimeException.class) @@ -146,6 +147,6 @@ public void testClassNotImplemented() throws SCMException { // set a placement class not implemented conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, "org.apache.hadoop.hdds.scm.container.placement.algorithm.HelloWorld"); - ContainerPlacementPolicyFactory.getPolicy(conf, null, null, true); + ContainerPlacementPolicyFactory.getPolicy(conf, null, null, true, null); } } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java index 2c58dc28fd..00ec3988e8 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java @@ -64,7 +64,8 @@ public void chooseDatanodes() throws SCMException { .thenReturn(new SCMNodeMetric(100L, 70L, 30L)); SCMContainerPlacementCapacity scmContainerPlacementRandom = - new SCMContainerPlacementCapacity(mockNodeManager, conf, null, true); + new SCMContainerPlacementCapacity(mockNodeManager, conf, null, true, + null); List existingNodes = new ArrayList<>(); existingNodes.add(datanodes.get(0)); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java index 3bd5f55945..b31152eaed 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java @@ -41,6 +41,8 @@ import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA; import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA; import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.when; @@ -58,6 +60,7 @@ public class TestSCMContainerPlacementRackAware { private SCMContainerPlacementRackAware policyNoFallback; // node storage capacity private static final long STORAGE_CAPACITY = 100L; + private SCMContainerPlacementMetrics metrics; @Before public void setup() { @@ -93,10 +96,11 @@ public void setup() { .thenReturn(new SCMNodeMetric(STORAGE_CAPACITY, 70L, 30L)); // create placement policy instances - policy = - new SCMContainerPlacementRackAware(nodeManager, conf, cluster, true); - policyNoFallback = - new SCMContainerPlacementRackAware(nodeManager, conf, cluster, false); + metrics = SCMContainerPlacementMetrics.create(); + policy = new SCMContainerPlacementRackAware( + nodeManager, conf, cluster, true, metrics); + policyNoFallback = new SCMContainerPlacementRackAware( + nodeManager, conf, cluster, false, metrics); } @@ -181,7 +185,6 @@ public void chooseNodeWithExcludedNodes() throws SCMException { @Test public void testFallback() throws SCMException { - // 5 replicas. there are only 3 racks. policy with fallback should // allocate the 5th datanode though it will break the rack rule(first // 2 replicas on same rack, others on different racks). @@ -195,14 +198,45 @@ public void testFallback() throws SCMException { datanodeDetails.get(2))); Assert.assertFalse(cluster.isSameParent(datanodeDetails.get(1), datanodeDetails.get(2))); + Assert.assertFalse(cluster.isSameParent(datanodeDetails.get(0), + datanodeDetails.get(3))); + Assert.assertFalse(cluster.isSameParent(datanodeDetails.get(2), + datanodeDetails.get(3))); + + // get metrics + long totalRequest = metrics.getDatanodeRequestCount(); + long successCount = metrics.getDatanodeChooseSuccessCount(); + long tryCount = metrics.getDatanodeChooseAttemptCount(); + long compromiseCount = metrics.getDatanodeChooseFallbackCount(); + + // verify metrics + Assert.assertTrue(totalRequest == nodeNum); + Assert.assertTrue(successCount == nodeNum); + Assert.assertTrue(tryCount > nodeNum); + Assert.assertTrue(compromiseCount >= 1); } - - @Test(expected = SCMException.class) + @Test public void testNoFallback() throws SCMException { // 5 replicas. there are only 3 racks. policy prohibit fallback should fail. int nodeNum = 5; - policyNoFallback.chooseDatanodes(null, null, nodeNum, 15); + try { + policyNoFallback.chooseDatanodes(null, null, nodeNum, 15); + fail("Fallback prohibited, this call should fail"); + } catch (Exception e) { + assertTrue(e.getClass().getSimpleName().equals("SCMException")); + } + + // get metrics + long totalRequest = metrics.getDatanodeRequestCount(); + long successCount = metrics.getDatanodeChooseSuccessCount(); + long tryCount = metrics.getDatanodeChooseAttemptCount(); + long compromiseCount = metrics.getDatanodeChooseFallbackCount(); + + Assert.assertTrue(totalRequest == nodeNum); + Assert.assertTrue(successCount >= 3); + Assert.assertTrue(tryCount >= nodeNum); + Assert.assertTrue(compromiseCount == 0); } @Test @@ -244,11 +278,28 @@ public void chooseNodeWithFavoredNodes() throws SCMException { .equals(favoredNodes.get(0).getNetworkFullPath())); } - @Test(expected = SCMException.class) + @Test public void testNoInfiniteLoop() throws SCMException { int nodeNum = 1; - // request storage space larger than node capability - policy.chooseDatanodes(null, null, nodeNum, STORAGE_CAPACITY + 15); + + try { + // request storage space larger than node capability + policy.chooseDatanodes(null, null, nodeNum, STORAGE_CAPACITY + 15); + fail("Storage requested exceeds capacity, this call should fail"); + } catch (Exception e) { + assertTrue(e.getClass().getSimpleName().equals("SCMException")); + } + + // get metrics + long totalRequest = metrics.getDatanodeRequestCount(); + long successCount = metrics.getDatanodeChooseSuccessCount(); + long tryCount = metrics.getDatanodeChooseAttemptCount(); + long compromiseCount = metrics.getDatanodeChooseFallbackCount(); + + Assert.assertTrue(totalRequest == nodeNum); + Assert.assertTrue(successCount == 0); + Assert.assertTrue(tryCount > nodeNum); + Assert.assertTrue(compromiseCount == 0); } @Test @@ -270,7 +321,8 @@ public void testDatanodeWithDefaultNetworkLocation() throws SCMException { // choose nodes to host 3 replica int nodeNum = 3; SCMContainerPlacementRackAware newPolicy = - new SCMContainerPlacementRackAware(nodeManager, conf, clusterMap, true); + new SCMContainerPlacementRackAware(nodeManager, conf, clusterMap, true, + metrics); List datanodeDetails = newPolicy.chooseDatanodes(null, null, nodeNum, 15); Assert.assertEquals(nodeNum, datanodeDetails.size()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java index adb30bcbec..43e3a8d134 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java @@ -59,7 +59,8 @@ public void chooseDatanodes() throws SCMException { .thenReturn(new SCMNodeMetric(100L, 90L, 10L)); SCMContainerPlacementRandom scmContainerPlacementRandom = - new SCMContainerPlacementRandom(mockNodeManager, conf, null, true); + new SCMContainerPlacementRandom(mockNodeManager, conf, null, true, + null); List existingNodes = new ArrayList<>(); existingNodes.add(datanodes.get(0)); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java index 62e1108f4a..f0b1cbb146 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java @@ -81,10 +81,10 @@ public void testCapacityPlacementYieldsBetterDataDistribution() throws SCMContainerPlacementCapacity capacityPlacer = new SCMContainerPlacementCapacity(nodeManagerCapacity, new Configuration(), - null, true); + null, true, null); SCMContainerPlacementRandom randomPlacer = new SCMContainerPlacementRandom(nodeManagerRandom, new Configuration(), - null, true); + null, true, null); for (int x = 0; x < opsCount; x++) { long containerSize = random.nextInt(100) * OzoneConsts.GB; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMContainerPlacementPolicyMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMContainerPlacementPolicyMetrics.java new file mode 100644 index 0000000000..536d807aed --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMContainerPlacementPolicyMetrics.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.scm; + +import org.apache.hadoop.hdds.HddsUtils; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.container.placement.algorithms + .SCMContainerPlacementMetrics; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.net.DNSToSwitchMapping; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.net.StaticMapping; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic + .NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY; +import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE; +import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; + +/** + * Test cases to verify the metrics exposed by SCMPipelineManager. + */ +public class TestSCMContainerPlacementPolicyMetrics { + + private MiniOzoneCluster cluster; + private MetricsRecordBuilder metrics; + private static OzoneClient ozClient = null; + private static ObjectStore store = null; + + @Before + public void setup() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, + "org.apache.hadoop.hdds.scm.container.placement.algorithms." + + "SCMContainerPlacementRackAware"); + conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true); + conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + StaticMapping.class, DNSToSwitchMapping.class); + StaticMapping.addNodeToRack(NetUtils.normalizeHostNames( + Collections.singleton(HddsUtils.getHostName(conf))).get(0), + "/rack1"); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(4) + .build(); + cluster.waitForClusterToBeReady(); + metrics = getMetrics(SCMContainerPlacementMetrics.class.getSimpleName()); + ozClient = OzoneClientFactory.getRpcClient(conf); + store = ozClient.getObjectStore(); + } + + /** + * Verifies container placement metric. + */ + @Test(timeout = 60000) + public void test() throws IOException { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + String value = "sample value"; + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + String keyName = UUID.randomUUID().toString(); + + // Write data into a key + try (OzoneOutputStream out = bucket.createKey(keyName, + value.getBytes().length, ReplicationType.RATIS, + THREE, new HashMap<>())) { + out.write(value.getBytes()); + } + + // close container + PipelineManager manager = + cluster.getStorageContainerManager().getPipelineManager(); + List pipelines = manager.getPipelines().stream().filter(p -> + p.getType() == HddsProtos.ReplicationType.RATIS && + p.getFactor() == HddsProtos.ReplicationFactor.THREE) + .collect(Collectors.toList()); + Pipeline targetPipeline = pipelines.get(0); + List nodes = targetPipeline.getNodes(); + manager.finalizeAndDestroyPipeline(pipelines.get(0), true); + + // kill datanode to trigger under-replicated container replication + cluster.shutdownHddsDatanode(nodes.get(0)); + try { + Thread.sleep(5 * 1000); + } catch (InterruptedException e) { + } + cluster.getStorageContainerManager().getReplicationManager() + .processContainersNow(); + try { + Thread.sleep(30 * 1000); + } catch (InterruptedException e) { + } + + long totalRequest = getLongCounter("DatanodeRequestCount", metrics); + long tryCount = getLongCounter("DatanodeChooseAttemptCount", metrics); + long sucessCount = + getLongCounter("DatanodeChooseSuccessCount", metrics); + long compromiseCount = + getLongCounter("DatanodeChooseFallbackCount", metrics); + + // Seems no under-replicated closed containers get replicated + Assert.assertTrue(totalRequest == 0); + Assert.assertTrue(tryCount == 0); + Assert.assertTrue(sucessCount == 0); + Assert.assertTrue(compromiseCount == 0); + } + + @After + public void teardown() { + cluster.shutdown(); + } +}