docker-compose.yml

version: "3.7"
services:
  hivemetastore:
    container_name: "hivemetastore"
    image: postgres:11.5
    hostname: hivemetastore
    environment:
      POSTGRES_PASSWORD: new_password
    expose:
      - 5432
    volumes:
      - ./docker/hive/init.sql:/docker-entrypoint-initdb.d/init.sql
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 20s
      timeout: 10s
      retries: 5
    # networks:
    #   spark_net:
    #     ipv4_address: 172.28.1.4
    # extra_hosts:
    #   - "master:172.28.1.1"
    #   - "worker1:172.28.1.2"
    #   - "worker2:172.28.1.3"
    #   - "zeppelin:172.28.1.5"
    #   - "livy:172.28.1.6"

  master:
    container_name: "master"
    # image: panovvv/hadoop-hive-spark:2.5
    build:
        context: .
        dockerfile: ./docker/master/Dockerfile
    hostname: master
    # depends_on:
    #   - hivemetastore
    environment:
      HADOOP_NODE: namenode
      HIVE_CONFIGURE: yes, please
      SPARK_PUBLIC_DNS: localhost
      SPARK_LOCAL_IP: 172.28.1.1
      SPARK_MASTER_HOST: 172.28.1.1
      SPARK_LOCAL_HOSTNAME: master
    expose:
      - 1-65535
    ports:
      # Spark Master Web UI
      - 8080:8080
      # Spark job Web UI: increments for each successive job
      - 4040:4040
      - 4041:4041
      - 4042:4042
      - 4043:4043
      # Spark History server
      - 18080:18080
      # YARN UI
      - 8088:8088
      # Hadoop namenode UI
      - 9870:9870
      # Hadoop secondary namenode UI
      - 9868:9868
      # Hive JDBC
      - 10000:10000
    volumes:
      - ./data:/data
    # networks:
    #   spark_net:
    #     ipv4_address: 172.28.1.1
    # extra_hosts:
    #   - "worker1:172.28.1.2"
    #   - "worker2:172.28.1.3"
    #   - "hivemetastore:172.28.1.4"
    #   - "zeppelin:172.28.1.5"
    #   - "livy:172.28.1.6"

  worker1:
    container_name: "worker1"
    # image: panovvv/hadoop-hive-spark:2.5
    build:
        context: .
        dockerfile: ./docker/worker/Dockerfile
    hostname: worker1
    # depends_on:
    #   - hivemetastore
    environment:
      SPARK_MASTER_ADDRESS: spark://master:7077
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8081
      SPARK_PUBLIC_DNS: localhost
      SPARK_LOCAL_HOSTNAME: worker1
      SPARK_LOCAL_IP: 172.28.1.2
      SPARK_MASTER_HOST: 172.28.1.1
      HADOOP_NODE: datanode
    expose:
      - 1-65535
    ports:
      # Hadoop datanode UI
      - 9864:9864
      #Spark worker UI
      - 8081:8081
    volumes:
      - ./data:/data
    # networks:
    #   spark_net:
    #     ipv4_address: 172.28.1.2
    # extra_hosts:
    #   - "master:172.28.1.1"
    #   - "worker2:172.28.1.3"
    #   - "hivemetastore:172.28.1.4"
    #   - "zeppelin:172.28.1.5"
    #   - "livy:172.28.1.6"

  worker2:
    container_name: "worker2"
    # image: panovvv/hadoop-hive-spark:2.5
    build:
        context: .
        dockerfile: ./docker/worker/Dockerfile
    hostname: worker2
    # depends_on:
    #   - hivemetastore
    environment:
      SPARK_MASTER_ADDRESS: spark://master:7077
      SPARK_WORKER_PORT: 8882
      SPARK_WORKER_WEBUI_PORT: 8082
      SPARK_PUBLIC_DNS: localhost
      SPARK_LOCAL_HOSTNAME: worker2
      SPARK_LOCAL_IP: 172.28.1.3
      SPARK_MASTER_HOST: 172.28.1.1
      HADOOP_NODE: datanode
      HADOOP_DATANODE_UI_PORT: 9865
    expose:
      - 1-65535
    ports:
      # Hadoop datanode UI
      - 9865:9865
      # Spark worker UI
      - 8082:8082
    volumes:
      - ./data:/data
    # networks:
    #   spark_net:
    #     ipv4_address: 172.28.1.3
    # extra_hosts:
    #   - "master:172.28.1.1"
    #   - "worker1:172.28.1.2"
    #   - "hivemetastore:172.28.1.4"
    #   - "zeppelin:172.28.1.5"
    #   - "livy:172.28.1.6"

  livy:
    container_name: "livy"
    build:
        context: .
        dockerfile: ./docker/livy/Dockerfile
    hostname: livy
    depends_on:
      - master
      - worker1
      - worker2
    volumes:
      - ./livy_batches:/livy_batches
      - ./data:/data
    environment:
      - SPARK_MASTER=yarn
      # Intentionally not specified - if it's set here, then we can't override it
      # via REST API ("conf"={} map)
      # Can be client or cluster
#      - SPARK_DEPLOY_MODE=client

      - LOCAL_DIR_WHITELIST=/data/batches/
      - ENABLE_HIVE_CONTEXT=false
      # Defaults are fine for variables below. Uncomment to change them.
#      - LIVY_HOST=0.0.0.0
#      - LIVY_PORT=8998
    expose:
      - 1-65535
    ports:
      - 8998:8998
    # networks:
    #   spark_net:
    #     ipv4_address: 172.28.1.6
    # extra_hosts:
    #   - "master:172.28.1.1"
    #   - "worker1:172.28.1.2"
    #   - "worker2:172.28.1.3"
    #   - "hivemetastore:172.28.1.4"
    #   - "zeppelin:172.28.1.5"

  zeppelin:
    container_name: "zeppelin"
    image: panovvv/zeppelin-bigdata:2.5
    hostname: zeppelin
    depends_on:
      - master
      - worker1
      - worker2
      - livy
    volumes:
      - ./zeppelin_notebooks:/zeppelin_notebooks
      - ./data:/data
    environment:
      ZEPPELIN_PORT: 8890
      ZEPPELIN_NOTEBOOK_DIR: '/zeppelin_notebooks'
    expose:
      - 8890
    ports:
      - 8890:8890
    # networks:
    #   spark_net:
    #     ipv4_address: 172.28.1.5
    # extra_hosts:
    #   - "master:172.28.1.1"
    #   - "worker1:172.28.1.2"
    #   - "worker2:172.28.1.3"
    #   - "hivemetastore:172.28.1.4"
    #   - "livy:172.28.1.6"
  postgres:
        image: postgres:9.6
        environment:
          - POSTGRES_USER=airflow
          - POSTGRES_PASSWORD=airflow
          - POSTGRES_DB=airflow
        ports:
          - "5433:5432"
        logging:
          options:
            max-size: 10m
            max-file: "3"
  scheduler:
    container_name: "scheduler"
    build:
      context: .
      dockerfile: ./docker/airflow/Dockerfile
    command: scheduler
    restart: always
    depends_on:
      - postgres
    env_file:
      - ./docker/airflow/.env
    volumes:
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
      - ./airflow/logs:/usr/local/airflow/logs
  webserver:
    container_name: "webserver"
    build:
        context: .
        dockerfile: ./docker/airflow/Dockerfile
    restart: always
    depends_on:
      - postgres
      - scheduler
    environment:
      - LOAD_EX=n
      - EXECUTOR=Local
      - PYSPARK_PYTHON=/usr/bin/python3
    volumes:
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
      - ./airflow/logs:/usr/local/airflow/logs  
      - ./airflow/dist:/usr/local/airflow/dist
    ports:
      - "9001:8080"
    command: webserver
    healthcheck:
      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
      interval: 30s
      timeout: 30s
      retries: 3         


# networks:
#   spark_net:
#     ipam:
#       driver: default
#       config:
#         - subnet: 172.28.0.0/16