From 34710c66d61bb09ffcd5c4cb3b9c92506b272720 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Fri, 1 Jun 2018 15:07:25 -0700 Subject: [PATCH] HDDS-143. Provide docker compose files to measure performance in a pseudo cluster. Contributed by Elek, Marton. --- hadoop-dist/src/main/compose/ozoneperf/.env | 17 ++++ .../src/main/compose/ozoneperf/README.md | 73 ++++++++++++++++++ .../src/main/compose/ozoneperf/compose-all.sh | 18 +++++ .../ozoneperf/docker-compose-freon.yaml | 26 +++++++ .../compose/ozoneperf/docker-compose.yaml | 77 +++++++++++++++++++ .../src/main/compose/ozoneperf/docker-config | 37 +++++++++ .../src/main/compose/ozoneperf/init.sh | 21 +++++ .../src/main/compose/ozoneperf/prometheus.yml | 24 ++++++ 8 files changed, 293 insertions(+) create mode 100644 hadoop-dist/src/main/compose/ozoneperf/.env create mode 100644 hadoop-dist/src/main/compose/ozoneperf/README.md create mode 100755 hadoop-dist/src/main/compose/ozoneperf/compose-all.sh create mode 100644 hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml create mode 100644 hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml create mode 100644 hadoop-dist/src/main/compose/ozoneperf/docker-config create mode 100755 hadoop-dist/src/main/compose/ozoneperf/init.sh create mode 100644 hadoop-dist/src/main/compose/ozoneperf/prometheus.yml diff --git a/hadoop-dist/src/main/compose/ozoneperf/.env b/hadoop-dist/src/main/compose/ozoneperf/.env new file mode 100644 index 0000000000..cac418ae59 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/.env @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=${hdds.version} diff --git a/hadoop-dist/src/main/compose/ozoneperf/README.md b/hadoop-dist/src/main/compose/ozoneperf/README.md new file mode 100644 index 0000000000..a78f208732 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/README.md @@ -0,0 +1,73 @@ + + +# Compose files for local performance tests + +This directory contains docker-compose definition for an ozone cluster where +all the metrics are saved to a prometheus instance. + + Prometheus follows a pull based approach where the metrics are published + on a HTTP endpoint. + + Our current approach: + + 1. A Java agent activates a prometheus metrics endpoint in every JVM instance + (use `init.sh` to download the agent) + + 2. The Java agent publishes all the jmx parameters in prometheus format AND + register the endpoint address to the consul. + + 3. Prometheus polls all the endpoints which are registered to consul. + + + +## How to use + +First of all download the required Java agent with running `./init.sh` + +After that you can start the cluster with docker-compose: + +``` +docker-compose up -d +``` + +After a while the cluster will be started. You can check the ozone web ui-s: + +https://localhost:9874 +https://localhost:9876 + +You can also scale up the datanodes: + +``` +docker-compose scale datanode=3 +``` + +Freon (Ozone test generator tool) is not part of docker-compose by default, +you can activate it using `compose-all.sh` instead of `docker-compose`: + +``` +compose-all.sh up -d +``` + +Now Freon is running. Let's try to check the metrics from the local Prometheus: + +http://localhost:9090/graph + +Example queries: + +``` +Hadoop_KeySpaceManager_NumKeyCommits +rate(Hadoop_KeySpaceManager_NumKeyCommits[10m]) +rate(Hadoop_Ozone_BYTES_WRITTEN[10m]) +``` diff --git a/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh b/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh new file mode 100755 index 0000000000..82ab8b3101 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker-compose -f docker-compose.yaml -f docker-compose-freon.yaml "$@" diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml new file mode 100644 index 0000000000..60bdc4a503 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + freon: + image: apache/hadoop-runner + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/ozone","freon"] diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml new file mode 100644 index 0000000000..fb7873bf88 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + namenode: + image: apache/hadoop-runner + hostname: namenode + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + ports: + - 9870:9870 + environment: + ENSURE_NAMENODE_DIR: /data/namenode + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/hdfs","namenode"] + datanode: + image: apache/hadoop-runner + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + ports: + - 9864 + command: ["/opt/hadoop/bin/ozone","datanode"] + env_file: + - ./docker-config + ksm: + image: apache/hadoop-runner + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + ports: + - 9874:9874 + environment: + ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/ozone","ksm"] + scm: + image: apache/hadoop-runner + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + ports: + - 9876:9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["/opt/hadoop/bin/ozone","scm"] + consul: + image: consul + command: ["agent", "-dev", "-ui", "-client", "0.0.0.0"] + ports: + - 8500:8500 + prometheus: + image: prom/prometheus + volumes: + - "./prometheus.yml:/etc/prometheus.yml" + command: ["--config.file","/etc/prometheus.yml"] + ports: + - 9090:9090 diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config new file mode 100644 index 0000000000..e4f5485ac5 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-config @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 +OZONE-SITE.XML_ozone.ksm.address=ksm +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=True +OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.handler.type=distributed +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 +HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +HADOOP_OPTS=-javaagent:/opt/jmxpromo.jar=port=0:consulHost=consul:consulMode=node + diff --git a/hadoop-dist/src/main/compose/ozoneperf/init.sh b/hadoop-dist/src/main/compose/ozoneperf/init.sh new file mode 100755 index 0000000000..cf25398bc2 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/init.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +EXPORTER_FILE="$DIR/jmxpromo.jar" +if [ ! -f "$EXPORTER_FILE" ]; then + wget https://github.com/flokkr/jmxpromo/releases/download/0.11/jmx_prometheus_javaagent-0.11.jar -O $EXPORTER_FILE +fi diff --git a/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml b/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml new file mode 100644 index 0000000000..80aa5203a2 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + +scrape_configs: + - job_name: jmxexporter + consul_sd_configs: + - server: consul:8500 + services: + - jmxexporter