docker-compose.yaml

services:
  # HADOOP
  hadoop-namenode:
    build: ./build/hadoop/namenode
    container_name: ${PROJECT_NAME}-hadoop-namenode
    hostname: ${PROJECT_NAME}-hadoop-namenode
    restart: unless-stopped
    environment:
      CLUSTER_NAME: ${PROJECT_NAME}-cluster
      VIRTUAL_HOST: ${HADOOP_SUBDOMAIN}  # define for reverse proxy
      VIRTUAL_PORT: 9870  # define internal port to forward to reverse proxy
      LETSENCRYPT_HOST: ${SPARK_SUBDOMAIN}  # define for encyption cert
      NETWORK_ACCESS: "internal"  # define to limit access to local network
    env_file:
      - ./build/hadoop/hadoop.env
    ports:
      #- $HADOOP_NAMENODE_HTTP_PORT:9870  # allow host/VPN access
      - $HADOOP_NAMENODE_IPC_PORT:9000  # allow host/VPN access
    volumes:
      - $HADOOP_NAMENODE_DIR:/hadoop/dfs/name
      - $SHARED_DIR:/opt/workspace
    networks:
      - "proxnet-ext"

  hadoop-datanode:
    build: ./build/hadoop/datanode
    container_name: ${PROJECT_NAME}-hadoop-datanode
    hostname: ${PROJECT_NAME}-hadoop-datanode
    restart: unless-stopped
    environment:
      SERVICE_PRECONDITION: "${PROJECT_NAME}-hadoop-namenode:9870"
    env_file:
      - ./build/hadoop/hadoop.env
    ports:
      - $HADOOP_DATANODE_PORT:9864  # allow host/VPN access
    volumes:
      - $HADOOP_DATANODE_DIR:/hadoop/dfs/data
      - $SHARED_DIR:/opt/workspace
    networks:
      - "proxnet-ext"
  
  hadoop-resourcemanager:
    build: ./build/hadoop/resourcemanager
    container_name: ${PROJECT_NAME}-hadoop-resourcemanager
    restart: unless-stopped
    environment:
      SERVICE_PRECONDITION: "${PROJECT_NAME}-hadoop-namenode:9000 ${PROJECT_NAME}-hadoop-namenode:9870 ${PROJECT_NAME}-hadoop-datanode:9864"
    env_file:
      - ./build/hadoop/hadoop.env
    networks:
      - "proxnet-ext"

  hadoop-nodemanager-1:
    build: ./build/hadoop/nodemanager
    container_name: ${PROJECT_NAME}-hadoop-nodemanager-1
    restart: unless-stopped
    environment:
      SERVICE_PRECONDITION: "${PROJECT_NAME}-hadoop-namenode:9000 ${PROJECT_NAME}-hadoop-namenode:9870 ${PROJECT_NAME}-hadoop-datanode:9864 ${PROJECT_NAME}-hadoop-resourcemanager:8088"
    env_file:
      - ./build/hadoop/hadoop.env
    networks:
      - "proxnet-ext"

  hadoop-historyserver:
    build: ./build/hadoop/historyserver
    container_name: ${PROJECT_NAME}-hadoop-historyserver
    restart: unless-stopped
    environment:
      SERVICE_PRECONDITION: "${PROJECT_NAME}-hadoop-namenode:9000 ${PROJECT_NAME}-hadoop-namenode:9870 ${PROJECT_NAME}-hadoop-datanode:9864 ${PROJECT_NAME}-hadoop-resourcemanager:8088"
    env_file:
      - ./build/hadoop/hadoop.env
    volumes:
      - $HADOOP_HISTORYSERVER_DIR:/hadoop/yarn/timeline
      - $SHARED_DIR:/opt/workspace
    networks:
      - "proxnet-ext"

  # SPARK
  spark-master:
    build: ./build/spark/master
    container_name: ${PROJECT_NAME}-spark-master
    restart: unless-stopped
    environment:
      VIRTUAL_HOST: ${SPARK_SUBDOMAIN}  # define for reverse proxy
      VIRTUAL_PORT: 8080  # define internal port to forward to reverse proxy
      LETSENCRYPT_HOST: ${SPARK_SUBDOMAIN}  # define for encryption cert
      NETWORK_ACCESS: "internal"  # define to limit access to local network
    ports:
      # - ${SPARK_MASTER_WEBUI_PORT}:8080
      #- 7077:7077  # allow docker network access
      - $SPARK_MASTER_PORT:7077  # allow host/VPN access to spark
      - $SPARK_CONNECT_PORT:15002  # allow host/VPN access to spark-connect
    volumes:
      - $SHARED_DIR:/opt/workspace
    networks:
      - "proxnet-ext"
    depends_on:
      - hadoop-namenode

  spark-worker-1:
    build: ./build/spark/worker
    container_name: ${PROJECT_NAME}-spark-worker-1
    restart: unless-stopped
    environment:
      - SPARK_WORKER_CORES=${SPARK_WORKER1_CORES}
      - SPARK_WORKER_MEMORY=${SPARK_WORKER1_MEMORY}
    ports:
      - $SPARK_WORKER1_WEBUI_PORT:8081  # 8081 defined in dockerfile
    volumes:
      - $SHARED_DIR:/opt/workspace
    networks:
      - "proxnet-ext"
    depends_on:
      - spark-master

  # JUPYTER
  jupyterlab:
    build: ./build/jupyter/lab
    container_name: ${PROJECT_NAME}-jupyter-lab
    restart: unless-stopped
    environment:
      VIRTUAL_HOST: ${JUPYTER_SUBDOMAIN}  # define for reverse proxy
      VIRTUAL_PORT: 8888  # define internal port to forward to reverse proxy
      LETSENCRYPT_HOST: ${JUPYTER_SUBDOMAIN}  # define for encryption cert
      NETWORK_ACCESS: "internal"  # define to limit access to local network
    #ports:  # define port alias if the host machine doesn't use a public ip
    #  - ${JUPYTER_PORT}:8888
    volumes:
      - $SHARED_DIR:/opt/workspace
    networks:
      - "proxnet-ext"


networks:
  proxnet-ext:
    external: true