HDDS-2166. Some RPC metrics are missing from SCM prometheus endpoint

Closes #1505

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/PrometheusMetricsSink.java

@@ -69,8 +69,10 @@ public void putMetrics(MetricsRecord metricsRecord) {
             .append(key)
             .append(" ")
             .append(metrics.type().toString().toLowerCase())
-            .append("\n")
-            .append(key)
+            .append("\n");
+
+        StringBuilder prometheusMetricKey = new StringBuilder();
+        prometheusMetricKey.append(key)
             .append("{");
         String sep = "";
 
@@ -80,7 +82,7 @@ public void putMetrics(MetricsRecord metricsRecord) {
 
           //ignore specific tag which includes sub-hierarchy
           if (!tagName.equals("numopenconnectionsperuser")) {
-            builder.append(sep)
+            prometheusMetricKey.append(sep)
                 .append(tagName)
                 .append("=\"")
                 .append(tag.value())
@@ -88,10 +90,14 @@ public void putMetrics(MetricsRecord metricsRecord) {
             sep = ",";
           }
         }
-        builder.append("} ");
+        prometheusMetricKey.append("}");
+
+        String prometheusMetricKeyAsString = prometheusMetricKey.toString();
+        builder.append(prometheusMetricKeyAsString);
+        builder.append(" ");
         builder.append(metrics.value());
         builder.append("\n");
-        metricLines.put(key, builder.toString());
+        metricLines.put(prometheusMetricKeyAsString, builder.toString());
 
       }
     }

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestPrometheusMetricsSink.java

@@ -21,17 +21,19 @@
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
 import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.MetricsTag;
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MutableCounterLong;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import org.junit.Assert;
 import org.junit.Test;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-
 /**
  * Test prometheus Sink.
  */
@@ -60,7 +62,6 @@ public void testPublish() throws IOException {
 
     //THEN
     String writtenMetrics = stream.toString(UTF_8.name());
-    System.out.println(writtenMetrics);
     Assert.assertTrue(
         "The expected metric line is missing from prometheus metrics output",
         writtenMetrics.contains(
@@ -71,6 +72,49 @@ public void testPublish() throws IOException {
     metrics.shutdown();
   }
 
+  @Test
+  public void testPublishWithSameName() throws IOException {
+    //GIVEN
+    MetricsSystem metrics = DefaultMetricsSystem.instance();
+
+    metrics.init("test");
+    PrometheusMetricsSink sink = new PrometheusMetricsSink();
+    metrics.register("Prometheus", "Prometheus", sink);
+    metrics.register("FooBar", "fooBar", (MetricsSource) (collector, all) -> {
+      collector.addRecord("RpcMetrics").add(new MetricsTag(PORT_INFO, "1234"))
+          .addGauge(COUNTER_INFO, 123).endRecord();
+
+      collector.addRecord("RpcMetrics").add(new MetricsTag(
+          PORT_INFO, "2345")).addGauge(COUNTER_INFO, 234).endRecord();
+    });
+
+    metrics.start();
+    metrics.publishMetricsNow();
+
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
+    OutputStreamWriter writer = new OutputStreamWriter(stream, UTF_8);
+
+    //WHEN
+    sink.writeMetrics(writer);
+    writer.flush();
+
+    //THEN
+    String writtenMetrics = stream.toString(UTF_8.name());
+    Assert.assertTrue(
+        "The expected metric line is missing from prometheus metrics output",
+        writtenMetrics.contains(
+            "rpc_metrics_counter{port=\"2345\""));
+
+    Assert.assertTrue(
+        "The expected metric line is missing from prometheus metrics "
+            + "output",
+        writtenMetrics.contains(
+            "rpc_metrics_counter{port=\"1234\""));
+
+    metrics.stop();
+    metrics.shutdown();
+  }
+
   @Test
   public void testNamingCamelCase() {
     PrometheusMetricsSink sink = new PrometheusMetricsSink();
@@ -127,4 +171,29 @@ private static class TestMetrics {
     @Metric
     private MutableCounterLong numBucketCreateFails;
   }
-}
+
+  public static final MetricsInfo PORT_INFO = new MetricsInfo() {
+    @Override
+    public String name() {
+      return "PORT";
+    }
+
+    @Override
+    public String description() {
+      return "port";
+    }
+  };
+
+  public static final MetricsInfo COUNTER_INFO = new MetricsInfo() {
+    @Override
+    public String name() {
+      return "COUNTER";
+    }
+
+    @Override
+    public String description() {
+      return "counter";
+    }
+  };
+
+}