From 993f36ee3341a32945ee54644fe8cc76065135a6 Mon Sep 17 00:00:00 2001 From: avijayanhwx <14299376+avijayanhwx@users.noreply.github.com> Date: Thu, 4 Apr 2019 06:48:32 -0700 Subject: [PATCH] HDDS-1353 : Metrics scm_pipeline_metrics_num_pipeline_creation_failed keeps increasing because of BackgroundPipelineCreator. (#681) --- .../InsufficientDatanodesException.java | 36 +++++++++++ .../scm/pipeline/RatisPipelineProvider.java | 2 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 2 + .../scm/pipeline/TestSCMPipelineManager.java | 61 +++++++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java new file mode 100644 index 0000000000..a6a5a69a16 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.pipeline; + +import java.io.IOException; + +/** + * Exception thrown when there are not enough Datanodes to create a pipeline. + */ +public class InsufficientDatanodesException extends IOException { + + + public InsufficientDatanodesException() { + super(); + } + + public InsufficientDatanodesException(String message) { + super(message); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java index 695220029a..6563e3fcaa 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java @@ -105,7 +105,7 @@ public Pipeline create(ReplicationFactor factor) throws IOException { String e = String .format("Cannot create pipeline of factor %d using %d nodes.", factor.getNumber(), dns.size()); - throw new IOException(e); + throw new InsufficientDatanodesException(e); } Pipeline pipeline = Pipeline.newBuilder() diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java index f274829df3..c72a52886c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java @@ -152,6 +152,8 @@ public synchronized Pipeline createPipeline( nodeManager.addPipeline(pipeline); metrics.incNumPipelineCreated(); return pipeline; + } catch (InsufficientDatanodesException idEx) { + throw idEx; } catch (IOException ex) { metrics.incNumPipelineCreationFailed(); throw ex; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java index 990d73ab90..53e968bf23 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hdds.scm.pipeline; +import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -30,6 +33,7 @@ import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.Assert; @@ -208,4 +212,61 @@ public void testPipelineReport() throws IOException { // clean up pipelineManager.close(); } + + @Test + public void testPipelineCreationFailedMetric() throws Exception { + MockNodeManager nodeManagerMock = new MockNodeManager(true, + 20); + SCMPipelineManager pipelineManager = + new SCMPipelineManager(conf, nodeManagerMock, new EventQueue()); + PipelineProvider mockRatisProvider = + new MockRatisPipelineProvider(nodeManagerMock, + pipelineManager.getStateManager(), conf); + pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, + mockRatisProvider); + + MetricsRecordBuilder metrics = getMetrics( + SCMPipelineMetrics.class.getSimpleName()); + long numPipelineCreated = getLongCounter("NumPipelineCreated", + metrics); + Assert.assertTrue(numPipelineCreated == 0); + + // 3 DNs are unhealthy. + // Create 5 pipelines (Use up 15 Datanodes) + for (int i = 0; i < 5; i++) { + Pipeline pipeline = pipelineManager + .createPipeline(HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE); + Assert.assertNotNull(pipeline); + } + + metrics = getMetrics( + SCMPipelineMetrics.class.getSimpleName()); + numPipelineCreated = getLongCounter("NumPipelineCreated", metrics); + Assert.assertTrue(numPipelineCreated == 5); + + long numPipelineCreateFailed = getLongCounter( + "NumPipelineCreationFailed", metrics); + Assert.assertTrue(numPipelineCreateFailed == 0); + + //This should fail... + try { + pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE); + Assert.fail(); + } catch (InsufficientDatanodesException idEx) { + Assert.assertEquals( + "Cannot create pipeline of factor 3 using 1 nodes.", + idEx.getMessage()); + } + + metrics = getMetrics( + SCMPipelineMetrics.class.getSimpleName()); + numPipelineCreated = getLongCounter("NumPipelineCreated", metrics); + Assert.assertTrue(numPipelineCreated == 5); + + numPipelineCreateFailed = getLongCounter( + "NumPipelineCreationFailed", metrics); + Assert.assertTrue(numPipelineCreateFailed == 0); + } }