HDFS-12983. Block Storage: provide docker-compose file for cblock clusters. Contributed by Elek,Marton.
This commit is contained in:
Anu Engineer
Owen O'Malley
parent
136b419594
commit
254cbe2589
17
dev-support/compose/cblock/.env
Normal file
17
dev-support/compose/cblock/.env
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
VERSION=3.1.0-SNAPSHOT
|
||||||
42
dev-support/compose/cblock/README.md
Normal file
42
dev-support/compose/cblock/README.md
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
|
||||||
|
# CBlock dozone configuration
|
||||||
|
|
||||||
|
This directory contains example cluster definition for CBlock/jscsi servers.
|
||||||
|
|
||||||
|
## How to use
|
||||||
|
|
||||||
|
1. First of all Start the servers with `docker-compose up -d`
|
||||||
|
|
||||||
|
2. Wait until the servers are up and running (check http://localhost:9876 and wait until you have a healthy node)
|
||||||
|
|
||||||
|
3. Create a volume: `docker-compose exec cblock hdfs cblock -c bilbo volume2 1GB 4`
|
||||||
|
|
||||||
|
4. Mount the iscsi volume (from host machine):
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo iscsiadm -m node -o new -T bilbo:volume2 -p 127.0.0.1
|
||||||
|
sudo iscsiadm -m node -T bilbo:volume2 --login
|
||||||
|
```
|
||||||
|
|
||||||
|
5. Check the device name from `dmesg` or `lsblk` (eg /dev/sdd). Errors in dmesg could be ignored: jscsi doesn't implement all the jscsi commands.
|
||||||
|
|
||||||
|
6. Format the device (`mkfs.ext4 /dev/sdd`). (Yes, format the while device, not just a partition).
|
||||||
|
|
||||||
|
7. Mount it (`mount /dev/sdd /mnt/target`).
|
||||||
64
dev-support/compose/cblock/docker-compose.yaml
Normal file
64
dev-support/compose/cblock/docker-compose.yaml
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
version: "3"
|
||||||
|
services:
|
||||||
|
namenode:
|
||||||
|
image: elek/hadoop-runner:latest
|
||||||
|
hostname: namenode
|
||||||
|
volumes:
|
||||||
|
- ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
|
||||||
|
ports:
|
||||||
|
- 9870:9870
|
||||||
|
environment:
|
||||||
|
ENSURE_NAMENODE_DIR: /data/namenode
|
||||||
|
env_file:
|
||||||
|
- ./docker-config
|
||||||
|
command: ["/opt/hadoop/bin/hdfs","namenode"]
|
||||||
|
datanode:
|
||||||
|
image: elek/hadoop-runner:latest
|
||||||
|
volumes:
|
||||||
|
- ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
|
||||||
|
ports:
|
||||||
|
- 9864
|
||||||
|
command: ["/opt/hadoop/bin/hdfs","datanode"]
|
||||||
|
jscsi:
|
||||||
|
image: elek/hadoop-runner:latest
|
||||||
|
ports:
|
||||||
|
- 3260:3260
|
||||||
|
volumes:
|
||||||
|
- ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
|
||||||
|
env_file:
|
||||||
|
- ./docker-config
|
||||||
|
command: ["/opt/hadoop/bin/hdfs","jscsi"]
|
||||||
|
cblock:
|
||||||
|
image: elek/hadoop-runner:latest
|
||||||
|
volumes:
|
||||||
|
- ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
|
||||||
|
env_file:
|
||||||
|
- ./docker-config
|
||||||
|
command: ["/opt/hadoop/bin/hdfs","cblockserver"]
|
||||||
|
scm:
|
||||||
|
image: elek/hadoop-runner:latest
|
||||||
|
volumes:
|
||||||
|
- ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
|
||||||
|
ports:
|
||||||
|
- 9876:9876
|
||||||
|
env_file:
|
||||||
|
- ./docker-config
|
||||||
|
command: ["/opt/hadoop/bin/hdfs","scm"]
|
||||||
|
environment:
|
||||||
|
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
|
||||||
38
dev-support/compose/cblock/docker-config
Normal file
38
dev-support/compose/cblock/docker-config
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
|
||||||
|
OZONE-SITE.XML_ozone.ksm.address=ksm
|
||||||
|
OZONE-SITE.XML_ozone.scm.names=scm
|
||||||
|
OZONE-SITE.XML_ozone.enabled=True
|
||||||
|
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
|
||||||
|
OZONE-SITE.XML_ozone.scm.block.client.address=scm
|
||||||
|
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
|
||||||
|
OZONE-SITE.XML_ozone.handler.type=distributed
|
||||||
|
OZONE-SITE.XML_ozone.scm.client.address=scm
|
||||||
|
|
||||||
|
OZONE-SITE.XML_dfs.cblock.jscsi.cblock.server.address=cblock
|
||||||
|
OZONE-SITE.XML_dfs.cblock.scm.ipaddress=scm
|
||||||
|
OZONE-SITE.XML_dfs.cblock.service.leveldb.path=/tmp
|
||||||
|
|
||||||
|
HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
|
||||||
|
HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
|
||||||
|
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
|
||||||
|
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
|
||||||
|
LOG4J.PROPERTIES_log4j.rootLogger=info,stdout
|
||||||
|
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||||
|
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||||
|
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
|
||||||
Loading…
Reference in New Issue
Block a user