diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..18daa50 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.python-version +generated/* +build-all.sh +build-conf.sh +deploy.sh +provision.sh diff --git a/README.md b/README.md index ec82102..d8475cf 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,40 @@ -# Druid Docker Image - -## Run a simple Druid cluster - -[Install Docker](docker-install.md) - -Download and launch the docker image - -```sh -docker pull druidio/example-cluster -docker run --rm -i -p 3000:8082 -p 3001:8081 -p 3090:8090 druidio/example-cluster -``` - -Wait a minute or so for Druid to start up and download the sample. - -On OS X - -- List datasources - -``` -curl http://$(docker-machine ip default):3000/druid/v2/datasources -``` - -- access the coordinator console - -``` -open http://$(docker-machine ip default):3001/ -``` - -On Linux - -- List datasources - -``` -curl http://localhost:3000/druid/v2/datasources -``` - -- access the coordinator console at http://localhost:3001/ - -## Build Druid Docker Image - -To build the docker image yourself - -```sh -git clone https://github.com/druid-io/docker-druid.git -docker build -t example-cluster docker-druid -``` +# Deploy Druid Cluster with Docker + +For an all-in-one Docker container(all Druid services running in one single container), look at [here](all-in-one/README.md) + +### Prerequisites: +1. Install docker and docker-machine on you development machine +1. Register a docker hub account if you don't have one. You are going to need to pull and push your docker images from there +1. Install Jinja2 using pip + `$ pip install jinja2` +1. For Mac OSX users only: install virtualbox + +### Directories: +1. `templates/scripts` has all the templates for docker management scripts: + - `build-all.sh.template`: the script to build all images and push to docker hub + - `build-conf.sh.template`: the script to build only the image that contains conf files + - `provision.sh.template`: the script to provision nodes + - `deploy.sh.template`: the script to deploy all nodes +1. `templates/dockerfiles` has all the Dockerfile templates that define images +1. `templates/conf` has all the configuration templates for druid nodes +1. When running `pre_build.py`, the Dockerfile templates and conf file templates will read `config.json` and render into `generated/` + **Always remember to change template files instead of those in the `generated/` directory as those will be overwritten once you run `pre_build.py`** + +### Usage: +1. For Mac OSX users only: Create a docker-machine dedicated for building images and setting up environment + `$ docker-machine create --driver virtualbox local && eval $(docker-machine env local)` +1. Change `config.json` accordingly +1. Prebuild + `$ python pre_build.py` +1. Run `$ ./provision.sh` to provision nodes +1. Build images accordingly by running either `build-all.sh` or `build-conf.sh` +1. Run `$ ./deploy.sh` + +### Container Lifecycle management: +1. To see the status of all containers, run `$ eval $(docker-machine env --swarm d-druid-swarm-master) && docker ps -a` + +### Notes: +1. To switch docker-machine, run `$ eval $(docker-machine env )`. + To switch to the swarm master, run `$ eval $(docker-machine env --swarm )` +1. To attach a new session to a running container, run `docker exec -it /bin/bash` +1. To view the logs for a druid node, run `docker logs `, or attach a new session and go to the actual log path to view the logs. diff --git a/Dockerfile b/all-in-one/Dockerfile similarity index 100% rename from Dockerfile rename to all-in-one/Dockerfile diff --git a/all-in-one/README.md b/all-in-one/README.md new file mode 100644 index 0000000..0b40f9a --- /dev/null +++ b/all-in-one/README.md @@ -0,0 +1,47 @@ +# All-In-One-Container Druid Docker Image + +## Run a simple Druid cluster + +[Install Docker](../docker-install.md) + +Download and launch the docker image + +```sh +docker pull druidio/example-cluster +docker run --rm -i -p 3000:8082 -p 3001:8081 -p 3090:8090 druidio/example-cluster +``` + +Wait a minute or so for Druid to start up and download the sample. + +On OS X + +- List datasources + +``` +curl http://$(docker-machine ip default):3000/druid/v2/datasources +``` + +- access the coordinator console + +``` +open http://$(docker-machine ip default):3001/ +``` + +On Linux + +- List datasources + +``` +curl http://localhost:3000/druid/v2/datasources +``` + +- access the coordinator console at http://localhost:3001/ + +## Build Druid Docker Image + +To build the docker image yourself + +```sh +git clone https://github.com/druid-io/docker-druid.git +cd all-in-one && docker build -t example-cluster docker-druid +``` diff --git a/sample-data.sql b/all-in-one/sample-data.sql similarity index 100% rename from sample-data.sql rename to all-in-one/sample-data.sql diff --git a/supervisord.conf b/all-in-one/supervisord.conf similarity index 100% rename from supervisord.conf rename to all-in-one/supervisord.conf diff --git a/config.json b/config.json new file mode 100644 index 0000000..d433478 --- /dev/null +++ b/config.json @@ -0,0 +1,25 @@ +{ + "docker_hub_username": "xiaoyao1991", + "image_tag": "latest", + "overlay_net": "d-my-net", + "common_conf_dir": "/usr/local/conf", + "deep_storage_dir": "/var/druid-segments", + + "coordinator_node_port": 8080, + "coordinator_node_aliases": ["coordinator"], + + "historical_node_port": 8081, + "historical_node_aliases": ["historical"], + + "broker_node_port": 8082, + "broker_node_aliases": ["broker"], + + "realtime_node_port": 8083, + "realtime_node_aliases": ["realtime"], + + "overlord_node_port": 8084, + "overlord_node_aliases": ["overlord"], + + "middle_manager_node_port": 8085, + "middle_manager_node_aliases": ["middlemanager"] +} diff --git a/pre_build.py b/pre_build.py new file mode 100644 index 0000000..ff1eca8 --- /dev/null +++ b/pre_build.py @@ -0,0 +1,55 @@ +import json +from jinja2 import Environment, FileSystemLoader +import os +import argparse +import stat +import errno + +TEMPLATE_DIR = "./templates/" +GENERATED_DIR = "./generated/" +CONFIG_FILE = "./config.json" + +def prebuild(): + # prepare jinja env + env = Environment(loader=FileSystemLoader('./')) + + # prepare user defined configs + with open(CONFIG_FILE, 'r') as fp: + config = json.load(fp) + + # generate directories + if not os.path.exists(GENERATED_DIR): + os.makedirs(GENERATED_DIR) + + os.chdir(TEMPLATE_DIR) + for dp, _, _ in os.walk('./'): + if not os.path.exists(os.path.join('../', GENERATED_DIR, dp)): + os.makedirs(os.path.join('../', GENERATED_DIR, dp)) + + # render all templates + templates = [os.path.join(dp, f) for dp, dn, filenames in os.walk('./') for f in filenames] + os.chdir('../') + + for template in templates: + generated_filename = GENERATED_DIR + template.rsplit(".template")[0] + with open(generated_filename, 'w') as fp: + if template.endswith('.template'): + fp.write(env.get_template(TEMPLATE_DIR + template).render(**config)) + else: + with open(TEMPLATE_DIR + template, 'r') as rofp: + fp.write(rofp.read()) + + scripts = ['build-all.sh', 'build-conf.sh', 'deploy.sh', 'provision.sh'] + for script in scripts: + try: + os.symlink(GENERATED_DIR + 'scripts/' + script, script) + except OSError, e: + if e.errno == errno.EEXIST: + os.remove(script) + os.symlink(GENERATED_DIR + 'scripts/' + script, script) + + st = os.stat(script) + os.chmod(script, st.st_mode | stat.S_IEXEC) + +if __name__ == '__main__': + prebuild() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7f7afbf --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +jinja2 diff --git a/templates/conf/druid/_common/common.runtime.properties.template b/templates/conf/druid/_common/common.runtime.properties.template new file mode 100644 index 0000000..e15ecff --- /dev/null +++ b/templates/conf/druid/_common/common.runtime.properties.template @@ -0,0 +1,114 @@ +# +# Licensed to Metamarkets Group Inc. (Metamarkets) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Metamarkets licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Extensions +# + +# This is not the full list of Druid extensions, but common ones that people often use. You may need to change this list +# based on your particular setup. +druid.extensions.loadList=["druid-kafka-eight", "druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "mysql-metadata-storage"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=zookeeper +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# druid.metadata.storage.type=derby +# druid.metadata.storage.connector.connectURI=jdbc:derby://metadata.store.ip:1527/var/druid/metadata.db;create=true +# druid.metadata.storage.connector.host=metadata.store.ip +# druid.metadata.storage.connector.port=1527 + +# For MySQL: +druid.metadata.storage.type=mysql +druid.metadata.storage.connector.connectURI=jdbc:mysql://mysql:3306/druid +druid.metadata.storage.connector.user=druid +druid.metadata.storage.connector.password=diurd + +# For PostgreSQL (make sure to additionally include the Postgres extension): +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +druid.storage.type=local +druid.storage.storageDirectory={{ deep_storage_dir }} + +# For HDFS (make sure to include the HDFS extension and that your Hadoop config files in the cp): +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS (make sure to include the HDFS extension and that your Hadoop config files in the cp): +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=hdfs://namenode.example.com:9000/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor"] +druid.emitter=logging +druid.emitter.logging.logLevel=info diff --git a/templates/conf/druid/_common/log4j2.xml.template b/templates/conf/druid/_common/log4j2.xml.template new file mode 100644 index 0000000..9798e9e --- /dev/null +++ b/templates/conf/druid/_common/log4j2.xml.template @@ -0,0 +1,19 @@ + + + + + + + + + %d %p %c{1.} [%t] %m%n + + + + + + + + + + diff --git a/templates/conf/druid/broker/jvm.config b/templates/conf/druid/broker/jvm.config new file mode 100644 index 0000000..a6a9982 --- /dev/null +++ b/templates/conf/druid/broker/jvm.config @@ -0,0 +1,8 @@ +-server +-Xms24g +-Xmx24g +-XX:MaxDirectMemorySize=4096m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/templates/conf/druid/broker/runtime.properties.template b/templates/conf/druid/broker/runtime.properties.template new file mode 100644 index 0000000..c8f721f --- /dev/null +++ b/templates/conf/druid/broker/runtime.properties.template @@ -0,0 +1,16 @@ +druid.service=druid/broker +druid.port={{ broker_node_port }} + +# HTTP server threads +druid.broker.http.numConnections=5 +druid.server.http.numThreads=25 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=536870912 +druid.processing.numThreads=7 + +# Query cache +druid.broker.cache.useCache=true +druid.broker.cache.populateCache=true +druid.cache.type=local +druid.cache.sizeInBytes=2000000000 diff --git a/templates/conf/druid/coordinator/jvm.config b/templates/conf/druid/coordinator/jvm.config new file mode 100644 index 0000000..99f6d1d --- /dev/null +++ b/templates/conf/druid/coordinator/jvm.config @@ -0,0 +1,8 @@ +-server +-Xms3g +-Xmx3g +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/templates/conf/druid/coordinator/runtime.properties.template b/templates/conf/druid/coordinator/runtime.properties.template new file mode 100644 index 0000000..fb27e23 --- /dev/null +++ b/templates/conf/druid/coordinator/runtime.properties.template @@ -0,0 +1,5 @@ +druid.service=druid/coordinator +druid.port={{ coordinator_node_port }} + +druid.coordinator.startDelay=PT30S +druid.coordinator.period=PT30S diff --git a/templates/conf/druid/historical/jvm.config b/templates/conf/druid/historical/jvm.config new file mode 100644 index 0000000..7fdbb1c --- /dev/null +++ b/templates/conf/druid/historical/jvm.config @@ -0,0 +1,8 @@ +-server +-Xms8g +-Xmx8g +-XX:MaxDirectMemorySize=4096m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/templates/conf/druid/historical/runtime.properties.template b/templates/conf/druid/historical/runtime.properties.template new file mode 100644 index 0000000..47b8cbd --- /dev/null +++ b/templates/conf/druid/historical/runtime.properties.template @@ -0,0 +1,13 @@ +druid.service=druid/historical +druid.port={{ historical_node_port }} + +# HTTP server threads +druid.server.http.numThreads=25 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=536870912 +druid.processing.numThreads=7 + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize"\:130000000000}] +druid.server.maxSize=130000000000 diff --git a/templates/conf/druid/middleManager/jvm.config b/templates/conf/druid/middleManager/jvm.config new file mode 100644 index 0000000..7b9f1a4 --- /dev/null +++ b/templates/conf/druid/middleManager/jvm.config @@ -0,0 +1,7 @@ +-server +-Xms64m +-Xmx64m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/templates/conf/druid/middleManager/runtime.properties.template b/templates/conf/druid/middleManager/runtime.properties.template new file mode 100644 index 0000000..ea30b89 --- /dev/null +++ b/templates/conf/druid/middleManager/runtime.properties.template @@ -0,0 +1,20 @@ +druid.service=druid/middleManager +druid.port={{ middle_manager_node_port }} + +# Number of tasks per middleManager +druid.worker.capacity=3 + +# Task launch parameters +druid.indexer.runner.javaOpts=-server -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=25 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=536870912 +druid.processing.numThreads=2 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp +druid.indexer.task.defaultHadoopCoordinates=["org.apache.hadoop:hadoop-client:2.3.0"] diff --git a/templates/conf/druid/overlord/jvm.config b/templates/conf/druid/overlord/jvm.config new file mode 100644 index 0000000..2df9a1c --- /dev/null +++ b/templates/conf/druid/overlord/jvm.config @@ -0,0 +1,7 @@ +-server +-Xms3g +-Xmx3g +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/templates/conf/druid/overlord/runtime.properties.template b/templates/conf/druid/overlord/runtime.properties.template new file mode 100644 index 0000000..4705cc8 --- /dev/null +++ b/templates/conf/druid/overlord/runtime.properties.template @@ -0,0 +1,7 @@ +druid.service=druid/overlord +druid.port={{ overlord_node_port }} + +druid.indexer.queue.startDelay=PT30S + +druid.indexer.runner.type=remote +druid.indexer.storage.type=metadata diff --git a/templates/conf/druid/realtime/runtime.properties.template b/templates/conf/druid/realtime/runtime.properties.template new file mode 100644 index 0000000..774d68c --- /dev/null +++ b/templates/conf/druid/realtime/runtime.properties.template @@ -0,0 +1,9 @@ +druid.service=druid/realtime +druid.port={{ realtime_node_port }} + +druid.processing.buffer.sizeBytes=1073741824 +druid.processing.numThreads=7 + +druid.server.http.numThreads=50 + +druid.monitoring.monitors=["io.druid.segment.realtime.RealtimeMetricsMonitor", "com.metamx.metrics.JvmMonitor"] diff --git a/templates/dockerfiles/Dockerfile-broker.template b/templates/dockerfiles/Dockerfile-broker.template new file mode 100644 index 0000000..ec31100 --- /dev/null +++ b/templates/dockerfiles/Dockerfile-broker.template @@ -0,0 +1,22 @@ +FROM {{ docker_hub_username }}/druid-base:{{ image_tag }} + +ENV CONTAINER_NAME broker + +RUN echo "[supervisord]\n\ +nodaemon=true\n\ +loglevel=debug\n\ +[program:druid-broker]\n\ +user=druid\n\ +command=java\n\ + -Xmx256m\n\ + -XX:MaxDirectMemorySize=4096m\n\ + -Dlogfilename=broker\n\ + -Duser.timezone=UTC\n\ + -Dfile.encoding=UTF-8\n\ + -classpath '{{ common_conf_dir }}/druid/_common:{{ common_conf_dir }}/druid/broker:lib/*'\n\ + io.druid.cli.Main server broker\n\ +priority=1" > /etc/supervisor/conf.d/supervisord.conf + +ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf + +EXPOSE {{ broker_node_port }} diff --git a/templates/dockerfiles/Dockerfile-conf.template b/templates/dockerfiles/Dockerfile-conf.template new file mode 100644 index 0000000..e8e1d14 --- /dev/null +++ b/templates/dockerfiles/Dockerfile-conf.template @@ -0,0 +1,7 @@ +FROM {{ docker_hub_username }}/ubuntu-java8:{{ image_tag }} + +# Place conf and scripts +ADD ./generated/conf/ {{ common_conf_dir }} + +# Mounting point for external configs and scripts +VOLUME ["{{ common_conf_dir }}"] diff --git a/templates/dockerfiles/Dockerfile-coordinator.template b/templates/dockerfiles/Dockerfile-coordinator.template new file mode 100644 index 0000000..79e0ae7 --- /dev/null +++ b/templates/dockerfiles/Dockerfile-coordinator.template @@ -0,0 +1,21 @@ +FROM {{ docker_hub_username }}/druid-base:{{ image_tag }} + +ENV CONTAINER_NAME coordinator + +RUN echo "[supervisord]\n\ +nodaemon=true\n\ +loglevel=debug\n\ +[program:druid-coordinator]\n\ +user=druid\n\ +command=java\n\ + -Xmx256m\n\ + -Dlogfilename=coordinator\n\ + -Duser.timezone=UTC\n\ + -Dfile.encoding=UTF-8\n\ + -classpath '{{ common_conf_dir }}/druid/_common:{{ common_conf_dir }}/druid/coordinator:lib/*'\n\ + io.druid.cli.Main server coordinator\n\ +priority=1" > /etc/supervisor/conf.d/supervisord.conf + +ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf + +EXPOSE {{ coordinator_node_port }} diff --git a/templates/dockerfiles/Dockerfile-druid-base.template b/templates/dockerfiles/Dockerfile-druid-base.template new file mode 100644 index 0000000..7818a2d --- /dev/null +++ b/templates/dockerfiles/Dockerfile-druid-base.template @@ -0,0 +1,52 @@ +FROM {{ docker_hub_username }}/ubuntu-java8:{{ image_tag }} + +# Maven +RUN wget -q -O - http://archive.apache.org/dist/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz | tar -xzf - -C /usr/local \ + && ln -s /usr/local/apache-maven-3.2.5 /usr/local/apache-maven \ + && ln -s /usr/local/apache-maven/bin/mvn /usr/local/bin/mvn + +# Druid system user +RUN adduser --system --group --no-create-home druid \ + && mkdir -p /var/lib/druid \ + && mkdir -p /var/log/druid \ + && mkdir -p /usr/local/druid \ + && chown druid:druid /var/lib/druid \ + && chown druid:druid /var/log/druid \ + && chown druid:druid /usr/local/druid + +# Druid (from source) +# whichever github owner (user or org name) you would like to build from +ENV GITHUB_OWNER druid-io +# whichever branch you would like to build +ENV DRUID_VERSION master + +# trigger rebuild only if branch changed +ADD https://api.github.com/repos/$GITHUB_OWNER/druid/git/refs/heads/$DRUID_VERSION druid-version.json +RUN git clone -q --branch $DRUID_VERSION --depth 1 https://github.com/$GITHUB_OWNER/druid.git /tmp/druid +WORKDIR /tmp/druid + +# package and install Druid locally +# use versions-maven-plugin 2.1 to work around https://jira.codehaus.org/browse/MVERSIONS-285 +RUN mvn -U -B org.codehaus.mojo:versions-maven-plugin:2.1:set -DgenerateBackupPoms=false -DnewVersion=$DRUID_VERSION \ + && mvn -U -B install -DskipTests=true -Dmaven.javadoc.skip=true \ + && cp distribution/target/druid-$DRUID_VERSION-bin.tar.gz /usr/local/druid \ + && cp distribution/target/mysql-metadata-storage-$DRUID_VERSION.tar.gz /usr/local/druid + +RUN cp -r distribution/target/extensions /usr/local/druid/ +RUN cp -r distribution/target/hadoop-dependencies /usr/local/druid/ + +WORKDIR /usr/local/druid +RUN tar --strip-components=1 -xvf druid-$DRUID_VERSION-bin.tar.gz + +# Manage extensions +RUN tar -xvf mysql-metadata-storage-$DRUID_VERSION.tar.gz -C extensions/ + +# clean up time +RUN apt-get purge --auto-remove -y git \ + && apt-get clean \ + && rm -rf /tmp/* \ + /var/tmp/* \ + /var/lib/apt/lists \ + /usr/local/apache-maven-3.2.5 \ + /usr/local/apache-maven \ + /root/.m2 diff --git a/templates/dockerfiles/Dockerfile-historical.template b/templates/dockerfiles/Dockerfile-historical.template new file mode 100644 index 0000000..f192d61 --- /dev/null +++ b/templates/dockerfiles/Dockerfile-historical.template @@ -0,0 +1,22 @@ +FROM {{ docker_hub_username }}/druid-base:{{ image_tag }} + +ENV CONTAINER_NAME historical + +RUN echo "[supervisord]\n\ +nodaemon=true\n\ +loglevel=debug\n\ +[program:druid-historical]\n\ +user=druid\n\ +command=java\n\ + -Xmx256m\n\ + -XX:MaxDirectMemorySize=4096m\n\ + -Dlogfilename=historical\n\ + -Duser.timezone=UTC\n\ + -Dfile.encoding=UTF-8\n\ + -classpath '{{ common_conf_dir }}/druid/_common:{{ common_conf_dir }}/druid/historical:lib/*'\n\ + io.druid.cli.Main server historical\n\ +priority=1" > /etc/supervisor/conf.d/supervisord.conf + +ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf + +EXPOSE {{ historical_node_port }} diff --git a/templates/dockerfiles/Dockerfile-middlemanager.template b/templates/dockerfiles/Dockerfile-middlemanager.template new file mode 100644 index 0000000..987cbfa --- /dev/null +++ b/templates/dockerfiles/Dockerfile-middlemanager.template @@ -0,0 +1,21 @@ +FROM {{ docker_hub_username }}/druid-base:{{ image_tag }} + +ENV CONTAINER_NAME middlemanager + +RUN echo "[supervisord]\n\ +nodaemon=true\n\ +loglevel=debug\n\ +[program:druid-middlemanager]\n\ +user=druid\n\ +command=java\n\ + -Xmx256m\n\ + -Dlogfilename=middlemanager\n\ + -Duser.timezone=UTC\n\ + -Dfile.encoding=UTF-8\n\ + -classpath '{{ common_conf_dir }}/druid/_common:{{ common_conf_dir }}/druid/middleManager:lib/*'\n\ + io.druid.cli.Main server middleManager\n\ +priority=1" > /etc/supervisor/conf.d/supervisord.conf + +ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf + +EXPOSE {{ middle_manager_node_port }} diff --git a/templates/dockerfiles/Dockerfile-overlord.template b/templates/dockerfiles/Dockerfile-overlord.template new file mode 100644 index 0000000..a5ffd7f --- /dev/null +++ b/templates/dockerfiles/Dockerfile-overlord.template @@ -0,0 +1,21 @@ +FROM {{ docker_hub_username }}/druid-base:{{ image_tag }} + +ENV CONTAINER_NAME overlord + +RUN echo "[supervisord]\n\ +nodaemon=true\n\ +loglevel=debug\n\ +[program:druid-overlord]\n\ +user=druid\n\ +command=java\n\ + -Xmx256m\n\ + -Dlogfilename=overlord\n\ + -Duser.timezone=UTC\n\ + -Dfile.encoding=UTF-8\n\ + -classpath '{{ common_conf_dir }}/druid/_common:{{ common_conf_dir }}/druid/overlord:lib/*'\n\ + io.druid.cli.Main server overlord\n\ +priority=1" > /etc/supervisor/conf.d/supervisord.conf + +ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf + +EXPOSE {{ overlord_node_port }} diff --git a/templates/dockerfiles/Dockerfile-realtime.template b/templates/dockerfiles/Dockerfile-realtime.template new file mode 100644 index 0000000..430458d --- /dev/null +++ b/templates/dockerfiles/Dockerfile-realtime.template @@ -0,0 +1,22 @@ +FROM {{ docker_hub_username }}/druid-base:{{ image_tag }} + +ENV CONTAINER_NAME realtime + +RUN echo "[supervisord]\n\ +nodaemon=true\n\ +loglevel=debug\n\ +[program:druid-realtime]\n\ +user=druid\n\ +command=java\n\ + -Xmx512m\n\ + -Dlogfilename=realtime\n\ + -Duser.timezone=UTC\n\ + -Dfile.encoding=UTF-8\n\ + -XX:MaxDirectMemorySize=4096m\n\ + -classpath '{{ common_conf_dir }}/druid/_common:{{ common_conf_dir }}/druid/realtime:lib/*'\n\ + io.druid.cli.Main server realtime\n\ +priority=1" > /etc/supervisor/conf.d/supervisord.conf + +ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf + +EXPOSE {{ realtime_node_port }} diff --git a/templates/dockerfiles/Dockerfile-ubuntu-java8 b/templates/dockerfiles/Dockerfile-ubuntu-java8 new file mode 100644 index 0000000..4650a90 --- /dev/null +++ b/templates/dockerfiles/Dockerfile-ubuntu-java8 @@ -0,0 +1,24 @@ +FROM ubuntu:14.04 + +# Java 8 +RUN apt-get update -y \ + && apt-get install -y software-properties-common \ + && apt-add-repository -y ppa:webupd8team/java \ + && apt-get purge --auto-remove -y software-properties-common \ + && apt-get update \ + && echo oracle-java-8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections \ + && apt-get install -y oracle-java8-installer \ + && apt-get install -y oracle-java8-set-default \ + && rm -rf /var/cache/oracle-jdk8-installer + +# Supervisor +RUN apt-get install -y supervisor + +# wget +RUN apt-get install -y wget + +# git +RUN apt-get install -y git + +# screen +RUN apt-get install -y screen diff --git a/templates/dockerfiles/Dockerfile-zookeeper.template b/templates/dockerfiles/Dockerfile-zookeeper.template new file mode 100644 index 0000000..e6a08ed --- /dev/null +++ b/templates/dockerfiles/Dockerfile-zookeeper.template @@ -0,0 +1,20 @@ +FROM {{ docker_hub_username }}/ubuntu-java8:{{ image_tag }} + +RUN mkdir -p /usr/local/dependencies +WORKDIR /usr/local/dependencies +RUN wget http://www-eu.apache.org/dist/zookeeper/stable/zookeeper-3.4.9.tar.gz \ + && tar -xf /usr/local/dependencies/zookeeper-3.4.9.tar.gz \ + && mv /usr/local/dependencies/zookeeper-3.4.9 /usr/local/dependencies/zookeeper \ + && cp /usr/local/dependencies/zookeeper/conf/zoo_sample.cfg /usr/local/dependencies/zookeeper/conf/zoo.cfg + +RUN echo "[supervisord]\n\ +nodaemon=true\n\ +loglevel=debug\n\ +[program:zookeeper]\n\ +user=daemon\n\ +command=/usr/local/dependencies/zookeeper/bin/zkServer.sh start-foreground\n\ +priority=1" > /etc/supervisor/conf.d/supervisord.conf + +ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf + +EXPOSE 2181 2888 3888 diff --git a/templates/scripts/build-all.sh.template b/templates/scripts/build-all.sh.template new file mode 100755 index 0000000..8399cf3 --- /dev/null +++ b/templates/scripts/build-all.sh.template @@ -0,0 +1,26 @@ +#!/bin/bash -e + +# Build docker images +docker build -f ./generated/dockerfiles/Dockerfile-ubuntu-java8 -t {{ docker_hub_username }}/ubuntu-java8:{{ image_tag }} . +docker build -f ./generated/dockerfiles/Dockerfile-conf -t {{ docker_hub_username }}/druid-conf:{{ image_tag }} . + +docker build -f ./generated/dockerfiles/Dockerfile-druid-base -t {{ docker_hub_username }}/druid-base:{{ image_tag }} . +docker build -f ./generated/dockerfiles/Dockerfile-broker -t {{ docker_hub_username }}/druid-broker:{{ image_tag }} . +docker build -f ./generated/dockerfiles/Dockerfile-coordinator -t {{ docker_hub_username }}/druid-coordinator:{{ image_tag }} . +docker build -f ./generated/dockerfiles/Dockerfile-historical -t {{ docker_hub_username }}/druid-historical:{{ image_tag }} . +docker build -f ./generated/dockerfiles/Dockerfile-middlemanager -t {{ docker_hub_username }}/druid-middlemanager:{{ image_tag }} . +docker build -f ./generated/dockerfiles/Dockerfile-overlord -t {{ docker_hub_username }}/druid-overlord:{{ image_tag }} . +docker build -f ./generated/dockerfiles/Dockerfile-realtime -t {{ docker_hub_username }}/druid-realtime:{{ image_tag }} . +docker build -f ./generated/dockerfiles/Dockerfile-zookeeper -t {{ docker_hub_username }}/druid-zookeeper:{{ image_tag }} . + +# Push docker images to hub +docker push {{ docker_hub_username }}/ubuntu-java8:{{ image_tag }} +docker push {{ docker_hub_username }}/druid-conf:{{ image_tag }} +docker push {{ docker_hub_username }}/druid-base:{{ image_tag }} +docker push {{ docker_hub_username }}/druid-zookeeper:{{ image_tag }} +docker push {{ docker_hub_username }}/druid-realtime:{{ image_tag }} +docker push {{ docker_hub_username }}/druid-overlord:{{ image_tag }} +docker push {{ docker_hub_username }}/druid-middlemanager:{{ image_tag }} +docker push {{ docker_hub_username }}/druid-historical:{{ image_tag }} +docker push {{ docker_hub_username }}/druid-coordinator:{{ image_tag }} +docker push {{ docker_hub_username }}/druid-broker:{{ image_tag }} diff --git a/templates/scripts/build-conf.sh.template b/templates/scripts/build-conf.sh.template new file mode 100644 index 0000000..df02716 --- /dev/null +++ b/templates/scripts/build-conf.sh.template @@ -0,0 +1,7 @@ +#!/bin/bash -e + +# Build conf and script images +docker build -f ./generated/dockerfiles/Dockerfile-conf -t {{docker_hub_username}}/druid-conf:{{ image_tag }} . + +# Push docker images to hub +docker push {{docker_hub_username}}/druid-conf:{{ image_tag }} diff --git a/templates/scripts/deploy.sh.template b/templates/scripts/deploy.sh.template new file mode 100755 index 0000000..d42fb0a --- /dev/null +++ b/templates/scripts/deploy.sh.template @@ -0,0 +1,16 @@ +#!/bin/bash -e + +eval $(docker-machine env --swarm d-druid-swarm-master) +docker run -itd --name=zookeeper --network={{ overlay_net }} --env="constraint:node==d-node-1" {{ docker_hub_username }}/druid-zookeeper:{{ image_tag }} +docker run -itd --name=mysql -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=druid -e MYSQL_USER=druid -e MYSQL_PASSWORD=diurd --network=d-my-net --env="constraint:node==d-node-2" mysql:latest --character-set-server=utf8 --collation-server=utf8_general_ci +for i in `seq 1 2`; +do + docker run -itd --name=node-$i-conf --network={{ overlay_net }} --env="constraint:node==d-node-$i" {{ docker_hub_username }}/druid-conf:{{ image_tag }} +done + +docker run -itd --volumes-from=node-1-conf -p {{ coordinator_node_port }}:{{ coordinator_node_port }} --name=coordinator --network={{ overlay_net }} --env="constraint:node==d-node-1" {{ docker_hub_username }}/druid-coordinator:{{ image_tag }} +docker run -itd --volumes-from=node-1-conf -p {{ overlord_node_port }}:{{ overlord_node_port }} --name=overlord --network={{ overlay_net }} --env="constraint:node==d-node-1" {{ docker_hub_username }}/druid-overlord:{{ image_tag }} +docker run -itd --volumes-from=node-1-conf --name=middlemanager --network={{ overlay_net }} --env="constraint:node==d-node-1" {{ docker_hub_username }}/druid-middlemanager:{{ image_tag }} +docker run -itd --volumes-from=node-2-conf --name=broker --network={{ overlay_net }} --env="constraint:node==d-node-2" {{ docker_hub_username }}/druid-broker:{{ image_tag }} +docker run -itd -v {{ deep_storage_dir }}:{{ deep_storage_dir }} --volumes-from=node-2-conf --name=realtime --network={{ overlay_net }} --env="constraint:node==d-node-2" {{ docker_hub_username }}/druid-realtime:{{ image_tag }} +docker run -itd -v {{ deep_storage_dir }}:{{ deep_storage_dir }} --volumes-from=node-2-conf --name=historical --network={{ overlay_net }} --env="constraint:node==d-node-2" {{ docker_hub_username }}/druid-historical:{{ image_tag }} diff --git a/templates/scripts/provision.sh.template b/templates/scripts/provision.sh.template new file mode 100755 index 0000000..b8fdd22 --- /dev/null +++ b/templates/scripts/provision.sh.template @@ -0,0 +1,36 @@ +#!/bin/bash -e + +# Create kv store for service discovery +docker-machine -D create --driver virtualbox d-mh-keystore + +eval $(docker-machine env d-mh-keystore) + +docker run -d \ + -p "8500:8500" \ + -h "consul" \ + progrium/consul -server -bootstrap + +# Create swarm master +docker-machine -D create --driver virtualbox \ + --swarm \ + --swarm-master \ + --swarm-discovery="consul://$(docker-machine ip d-mh-keystore):8500" \ + --engine-opt="cluster-store=consul://$(docker-machine ip d-mh-keystore):8500" \ + --engine-opt="cluster-advertise=eth1:2376" \ + d-druid-swarm-master + +# Create swarm workers +for i in `seq 1 2`; +do + echo $i + docker-machine -D create --driver virtualbox \ + --swarm \ + --swarm-discovery="consul://$(docker-machine ip d-mh-keystore):8500" \ + --engine-opt="cluster-store=consul://$(docker-machine ip d-mh-keystore):8500" \ + --engine-opt="cluster-advertise=eth1:2376" \ + d-node-$i +done + +# Create overlay network +eval $(docker-machine env --swarm d-druid-swarm-master) +docker network create --driver overlay --subnet=10.0.9.0/24 {{ overlay_net }}