From 94090acf3a61220142ec134b4f9f59f5165c70b3 Mon Sep 17 00:00:00 2001
From: Anu Engineer <aengineer@apache.org>
Date: Wed, 27 Sep 2017 12:04:09 -0700
Subject: [PATCH] HDFS-12469. Ozone: Create docker-compose definition to easily
 test real clusters. Contributed by Elek, Marton.

---
 dev-support/compose/ozone/.env                |  1 +
 dev-support/compose/ozone/docker-compose.yaml | 40 +++++++++++++++++
 dev-support/compose/ozone/docker-config       | 17 +++++++
 .../site/markdown/OzoneGettingStarted.md.vm   | 45 +++++++++++++++++++
 4 files changed, 103 insertions(+)
 create mode 100644 dev-support/compose/ozone/.env
 create mode 100644 dev-support/compose/ozone/docker-compose.yaml
 create mode 100644 dev-support/compose/ozone/docker-config

diff --git a/dev-support/compose/ozone/.env b/dev-support/compose/ozone/.env
new file mode 100644
index 0000000000..948e2d5d23
--- /dev/null
+++ b/dev-support/compose/ozone/.env
@@ -0,0 +1 @@
+VERSION=3.1.0-SNAPSHOT
diff --git a/dev-support/compose/ozone/docker-compose.yaml b/dev-support/compose/ozone/docker-compose.yaml
new file mode 100644
index 0000000000..1b89310d60
--- /dev/null
+++ b/dev-support/compose/ozone/docker-compose.yaml
@@ -0,0 +1,40 @@
+version: "3"
+services:
+   namenode:
+      image: flokkr/hadoop-runner:latest
+      hostname: namenode
+      volumes:
+         - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+      ports:
+         - 50070:50070
+         - 9870:9870
+      environment:
+          ENSURE_NAMENODE_DIR: /data/namenode
+      env_file:
+         - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","namenode"]
+   datanode:
+      image: flokkr/hadoop-runner:latest
+      volumes:
+        - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+      ports:
+        - 9864
+      command: ["/opt/hadoop/bin/hdfs","datanode"]
+   ksm:
+      image: flokkr/hadoop-runner:latest
+      volumes:
+         - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+      ports:
+         - 9874:9874
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","ksm"]
+   scm:
+      image: flokkr/hadoop-runner:latest
+      volumes:
+         - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+      ports:
+         - 9876:9876
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","scm"]
diff --git a/dev-support/compose/ozone/docker-config b/dev-support/compose/ozone/docker-config
new file mode 100644
index 0000000000..72e95a2bda
--- /dev/null
+++ b/dev-support/compose/ozone/docker-config
@@ -0,0 +1,17 @@
+CORE-SITE.XML_fs.defaultFS: "hdfs://namenode:9000"
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
index b9abd8484e..ec9d65ccb3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
@@ -41,9 +41,54 @@ example of the tarball that will be generated.
 
 * `~/apache/hadoop/hadoop-dist/target/${project.version}.tar.gz`
 
+At this point we have an option to setup a physical cluster or run ozone via
+docker.
+
+Running Ozone via Docker
+------------------------
+
+This assumes that you have a running docker setup on the machine. Please run
+these following commands to see ozone in action.
+
+ Go to the directory where the docker compose files exist.
+
+
+ - `cd dev-support/compose/ozone`
+
+Tell docker to start ozone, this will start a KSM, SCM and a single datanode in
+the background.
+
+
+ - `docker-compose up -d`
+
+Now let us run some work load against ozone, to do that we will run corona.
+
+This will log into the datanode and run bash.
+
+ - `docker-compose exec datanode bash`
+ - `cd hadoop/bin`
+
+Now you can run the oz command shell or corona the ozone load generator.
+
+This is the command to run corona.
+
+ - `./hdfs corona -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100`
+
+You can checkout the KSM UI to see the requests information.
+
+ - `http://localhost:9874/`
+
+If you need more datanode you can scale up:
+
+ - `docker-compose scale datanode=3`
+
+Running Ozone using a real cluster
+----------------------------------
+
 Please proceed to setup a hadoop cluster by creating the hdfs-site.xml and
 other configuration files that are needed for your cluster.
 
+
 ### Ozone Configuration
 
 Ozone relies on its own configuration file called `ozone-site.xml`. It is