diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 0fa78c2f2a..7e3e47c80d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -961,5 +961,13 @@ public class CommonConfigurationKeysPublic { /** Default shutdown hook timeout: {@value} seconds. */ public static final long SERVICE_SHUTDOWN_TIMEOUT_DEFAULT = 30; + /** + * @see + * + * core-default.xml + */ + public static final String HADOOP_PROMETHEUS_ENABLED = + "hadoop.prometheus.endpoint.enabled"; + public static final boolean HADOOP_PROMETHEUS_ENABLED_DEFAULT = false; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 7825e08dac..82d7b59dc7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -61,8 +61,11 @@ import org.apache.hadoop.conf.ConfServlet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration.IntegerRanges; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.jmx.JMXJsonServlet; import org.apache.hadoop.log.LogLevel; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.sink.PrometheusMetricsSink; import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -191,6 +194,11 @@ public final class HttpServer2 implements FilterContainer { private static final String X_FRAME_OPTIONS = "X-FRAME-OPTIONS"; private static final Pattern PATTERN_HTTP_HEADER_REGEX = Pattern.compile(HTTP_HEADER_REGEX); + + private boolean prometheusSupport; + protected static final String PROMETHEUS_SINK = "PROMETHEUS_SINK"; + private PrometheusMetricsSink prometheusMetricsSink; + /** * Class to construct instances of HTTP server with specific options. */ @@ -612,6 +620,19 @@ public final class HttpServer2 implements FilterContainer { } addDefaultServlets(); + addPrometheusServlet(conf); + } + + private void addPrometheusServlet(Configuration conf) { + prometheusSupport = conf.getBoolean( + CommonConfigurationKeysPublic.HADOOP_PROMETHEUS_ENABLED, + CommonConfigurationKeysPublic.HADOOP_PROMETHEUS_ENABLED_DEFAULT); + if (prometheusSupport) { + prometheusMetricsSink = new PrometheusMetricsSink(); + getWebAppContext().getServletContext() + .setAttribute(PROMETHEUS_SINK, prometheusMetricsSink); + addServlet("prometheus", "/prom", PrometheusServlet.class); + } } private void addListener(ServerConnector connector) { @@ -1133,6 +1154,11 @@ public final class HttpServer2 implements FilterContainer { try { openListeners(); webServer.start(); + if (prometheusSupport) { + DefaultMetricsSystem.instance() + .register("prometheus", "Hadoop metrics prometheus exporter", + prometheusMetricsSink); + } } catch (IOException ex) { LOG.info("HttpServer.start() threw a non Bind IOException", ex); throw ex; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/PrometheusServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/PrometheusServlet.java new file mode 100644 index 0000000000..abcc088a03 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/PrometheusServlet.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.http; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.sink.PrometheusMetricsSink; + +/** + * Servlet to publish hadoop metrics in prometheus format. + */ +public class PrometheusServlet extends HttpServlet { + + public PrometheusMetricsSink getPrometheusSink() { + return + (PrometheusMetricsSink) getServletContext().getAttribute( + HttpServer2.PROMETHEUS_SINK); + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + DefaultMetricsSystem.instance().publishMetricsNow(); + getPrometheusSink().writeMetrics(resp.getWriter()); + resp.getWriter().flush(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java new file mode 100644 index 0000000000..291cfe3295 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.sink; + +import org.apache.commons.configuration2.SubsetConfiguration; +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricType; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.MetricsSink; +import org.apache.hadoop.metrics2.MetricsTag; + +import java.io.IOException; +import java.io.Writer; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +import org.apache.commons.lang3.StringUtils; + +/** + * Metrics sink for prometheus exporter. + *
+ * Stores the metric data in-memory and return with it on request.
+ */
+public class PrometheusMetricsSink implements MetricsSink {
+
+ /**
+ * Cached output lines for each metrics.
+ */
+ private Map