docker-compose.yml

# http://github.com/yeasy/docker-compose-files
# This compose file will start spark master node and the worker node.
# All nodes will become a cluster automatically.
# You can run: docker-compose scale worker=2
# After startup, try submit a pi calculation application.
#  /usr/local/spark/bin/spark-submit --master spark://master:7077 --class org.apache.spark.examples.SparkPi /usr/local/spark/lib/spark-examples-1.4.0-hadoop2.6.0.jar 1000

version: "3.9"
services:
  master:
    image: sequenceiq/spark:1.6.0
    hostname: master
    ports:
    - "4040:4040"
    - "8042:8042"
    - "7077:7077"
    - "8088:8088"
    - "8080:8080"
    restart: always
    #mem_limit: 1024m
    command: bash /usr/local/spark/sbin/start-master.sh && ping localhost > /dev/null

  worker:
    image: sequenceiq/spark:1.6.0
    links:
    - master:master
    expose:
    - "8081"
    restart: always
    command: bash /usr/local/spark/sbin/start-slave.sh spark://master:7077 && ping localhost >/dev/null