docker-compose.yml

version: "3.9"
services:
  notebook:
    build:
      context: .
      dockerfile: ./Dockerfile
      target: notebook
    ports:
      - "8889:8889"
    volumes:
      - .:/app/:rw

  spark:
    build:
      context: .
      dockerfile: ./Dockerfile
      target: apache-spark
    ports:
      - "9090:8080"
      - "7077:7077"
    environment:
      SPARK_MODE: master
      SPARK_MASTER_PORT: 7077
      SPARK_MASTER_WEBUI_PORT: 8080
    volumes:
      - .:/app/:rw

  spark-worker:
    build:
      context: .
      dockerfile: ./Dockerfile
      target: apache-spark
    ports:
      - "9091:9091"
      - "7000:7000"
    depends_on:
      - spark
    environment:
      SPARK_MODE: worker
      SPARK_MASTER_URL: spark://spark:7077
      # see https://spark.apache.org/docs/latest/spark-standalone.html#cluster-launch-scripts 
      # for the following environment variables meaning
      SPARK_WORKER_PORT: 7000
      SPARK_WORKER_WEBUI_PORT: 9091
      SPARK_WORKER_CORES: 1
      SPARK_WORKER_MEMORY: 1G
    volumes:
      - .:/app/:rw