diff --git a/.env.local.example b/.env.local.example index 3f9b15eb..dc5461f7 100644 --- a/.env.local.example +++ b/.env.local.example @@ -13,7 +13,7 @@ LAMASSU_DB_CONNECT_API_KEY=secret DEER_USER= DEER_PASSWORD= -# A random password for pgbouncer clients (gtfs-api, geoserver, etc.) +# A random password for pgbouncer clients (gtfs-api, geoserver, dagster, etc.) PGBOUNCER_POSTGRES_PASSWORD=XP1EdA6S0BHmybkC # A random password which will be used for the postgresql database which is used by Dagster. Should be long and should not contain special diff --git a/docker-compose.yml b/docker-compose.yml index b0226e53..580c775d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -175,10 +175,12 @@ services: image: ${DAGSTER_PIPELINE_IMAGE} restart: always environment: - - PGHOST=dagster-postgresql - - PGUSER=${DAGSTER_POSTGRES_USER} - - PGPASSWORD=${DAGSTER_POSTGRES_PASSWORD} - - PGDATABASE=${DAGSTER_POSTGRES_DB} + # connect via pgbouncer to improve performance + - PGHOST=pgbouncer + - PGPORT=${PGBOUNCER_POSTGRES_PORT:?missing/empty $PGBOUNCER_POSTGRES_PORT} + - PGUSER=${PGBOUNCER_POSTGRES_USER:?missing/empty $PGBOUNCER_POSTGRES_USER} + - PGPASSWORD=${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty $PGBOUNCER_POSTGRES_PASSWORD} + - PGDATABASE=dagster # determined by pgbouncer! - IPL_LAMASSU_BASE_URL=${LAMASSU_BASE_URL} - IPL_POSTGRES_HOST=ipl-db - IPL_POSTGRES_PORT @@ -186,7 +188,7 @@ services: - IPL_POSTGRES_USER - IPL_POSTGRES_PASSWORD depends_on: - dagster-postgresql: + pgbouncer: condition: service_healthy # This service runs dagster-webserver, which loads your user code from the user code container. @@ -200,16 +202,19 @@ services: ports: - "3000:3000" environment: - PGHOST: dagster-postgresql - PGUSER: ${DAGSTER_POSTGRES_USER} - PGPASSWORD: ${DAGSTER_POSTGRES_PASSWORD} - PGDATABASE: ${DAGSTER_POSTGRES_DB} + # connect via pgbouncer to improve performance + PGHOST: pgbouncer + # todo: doesn't work yet, uses default port 5432 instead + PGPORT: ${PGBOUNCER_POSTGRES_PORT:?missing/empty $PGBOUNCER_POSTGRES_PORT} + PGUSER: ${PGBOUNCER_POSTGRES_USER:?missing/empty $PGBOUNCER_POSTGRES_USER} + PGPASSWORD: ${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty $PGBOUNCER_POSTGRES_PASSWORD} + PGDATABASE: dagster # determined by pgbouncer! volumes: # Make docker client accessible so we can terminate containers from dagster-webserver - /var/run/docker.sock:/var/run/docker.sock - /tmp/io_manager_storage:/tmp/io_manager_storage - ./etc/dagster:/opt/dagster/dagster_home/ depends_on: - dagster-postgresql: + pgbouncer: condition: service_healthy dagster-pipeline: condition: service_started @@ -221,16 +226,19 @@ services: image: ${DAGSTER_DAEMON_IMAGE} restart: on-failure environment: - PGHOST: dagster-postgresql - PGUSER: ${DAGSTER_POSTGRES_USER} - PGPASSWORD: ${DAGSTER_POSTGRES_PASSWORD} - PGDATABASE: ${DAGSTER_POSTGRES_DB} + # connect via pgbouncer to improve performance + PGHOST: pgbouncer + # todo: doesn't work yet, uses default port 5432 instead + PGPORT: ${PGBOUNCER_POSTGRES_PORT:?missing/empty $PGBOUNCER_POSTGRES_PORT} + PGUSER: ${PGBOUNCER_POSTGRES_USER:?missing/empty $PGBOUNCER_POSTGRES_USER} + PGPASSWORD: ${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty $PGBOUNCER_POSTGRES_PASSWORD} + PGDATABASE: dagster # determined by pgbouncer! volumes: # Make docker client accessible so we can launch containers using host docker - /var/run/docker.sock:/var/run/docker.sock - /tmp/io_manager_storage:/tmp/io_manager_storage - ./etc/dagster:/opt/dagster/dagster_home/ depends_on: - dagster-postgresql: + pgbouncer: condition: service_healthy dagster-pipeline: condition: service_started @@ -253,7 +261,7 @@ services: start_period: 10s retries: 10 - # pgbouncer acts like a facade in front of PostgreSQL instances (ipl-db, gtfs-db), providing + # pgbouncer acts like a facade in front of PostgreSQL instances (ipl-db, gtfs-db, dagster-postgresql), providing # - programmatically configurable query routing (used by `make import-new-gtfs`) # - improved connect performance for clients, especially Dagster workers pgbouncer: @@ -263,6 +271,8 @@ services: condition: service_started gtfs-db: condition: service_started + dagster-postgresql: + condition: service_started links: - ipl-db - gtfs-db @@ -300,7 +310,8 @@ services: # see also https://www.postgresql.org/docs/15/libpq-connect.html#id-1.7.3.8.3.5 # expose ipl-db as `ipl` PGBOUNCER_DSN_0: "ipl=host=ipl-db dbname=${IPL_POSTGRES_DB:?missing/empty $IPL_POSTGRES_DB} user=${IPL_POSTGRES_USER:?missing/empty $IPL_POSTGRES_USER} password=${IPL_POSTGRES_PASSWORD:?missing/empty $IPL_POSTGRES_PASSWORD}" - # todo: expose dagster-postgresql as `dagster` + # expose dagster-postgresql as `dagster` + PGBOUNCER_DSN_1: "dagster=host=dagster-postgresql dbname=${DAGSTER_POSTGRES_DB:?missing/empty $DAGSTER_POSTGRES_DB} user=${DAGSTER_POSTGRES_USER:?missing/empty $DAGSTER_POSTGRES_USER} password=${DAGSTER_POSTGRES_PASSWORD:?missing/empty $DAGSTER_POSTGRES_PASSWORD}" # expose the latest GTFS import within gtfs-db as `gtfs` # $PGBOUNCER_DSN_2 gets generated from /var/pgbouncer-dsn.txt (which is written by gtfs-importer) by the Docker image's entrypoint script. # see https://github.com/bitnami/containers/issues/46152#issuecomment-1695320501