let dagster-* connect to pgbouncer instead of dagster-postgresql

mobidata-bw · Oct 12, 2023 · a33dfd9 · a33dfd9
1 parent cb67cef
commit a33dfd9
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 18 deletions.
diff --git a/.env.local.example b/.env.local.example
@@ -13,7 +13,7 @@ LAMASSU_DB_CONNECT_API_KEY=secret
 DEER_USER=
 DEER_PASSWORD=
 
-# A random password for pgbouncer clients (gtfs-api, geoserver, etc.)
+# A random password for pgbouncer clients (gtfs-api, geoserver, dagster, etc.)
 PGBOUNCER_POSTGRES_PASSWORD=XP1EdA6S0BHmybkC
 
 # A random password which will be used for the postgresql database which is used by Dagster. Should be long and should not contain special

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -175,18 +175,20 @@ services:
     image: ${DAGSTER_PIPELINE_IMAGE}
     restart: always
     environment:
-      - PGHOST=dagster-postgresql
-      - PGUSER=${DAGSTER_POSTGRES_USER}
-      - PGPASSWORD=${DAGSTER_POSTGRES_PASSWORD}
-      - PGDATABASE=${DAGSTER_POSTGRES_DB}
+      # connect via pgbouncer to improve performance
+      - PGHOST=pgbouncer
+      - PGPORT=${PGBOUNCER_POSTGRES_PORT:?missing/empty $PGBOUNCER_POSTGRES_PORT}
+      - PGUSER=${PGBOUNCER_POSTGRES_USER:?missing/empty $PGBOUNCER_POSTGRES_USER}
+      - PGPASSWORD=${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty $PGBOUNCER_POSTGRES_PASSWORD}
+      - PGDATABASE=dagster # determined by pgbouncer!
       - IPL_LAMASSU_BASE_URL=${LAMASSU_BASE_URL}
       - IPL_POSTGRES_HOST=ipl-db
       - IPL_POSTGRES_PORT
       - IPL_POSTGRES_DB
       - IPL_POSTGRES_USER
       - IPL_POSTGRES_PASSWORD
     depends_on:
-        dagster-postgresql:
+        pgbouncer:
           condition: service_healthy
 
   # This service runs dagster-webserver, which loads your user code from the user code container.
@@ -200,16 +202,19 @@ services:
     ports:
       - "3000:3000"
     environment:
-      PGHOST: dagster-postgresql
-      PGUSER: ${DAGSTER_POSTGRES_USER}
-      PGPASSWORD: ${DAGSTER_POSTGRES_PASSWORD}
-      PGDATABASE: ${DAGSTER_POSTGRES_DB}
+      # connect via pgbouncer to improve performance
+      PGHOST: pgbouncer
+      # todo: doesn't work yet, uses default port 5432 instead
+      PGPORT: ${PGBOUNCER_POSTGRES_PORT:?missing/empty $PGBOUNCER_POSTGRES_PORT}
+      PGUSER: ${PGBOUNCER_POSTGRES_USER:?missing/empty $PGBOUNCER_POSTGRES_USER}
+      PGPASSWORD: ${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty $PGBOUNCER_POSTGRES_PASSWORD}
+      PGDATABASE: dagster # determined by pgbouncer!
     volumes: # Make docker client accessible so we can terminate containers from dagster-webserver
       - /var/run/docker.sock:/var/run/docker.sock
       - /tmp/io_manager_storage:/tmp/io_manager_storage
       - ./etc/dagster:/opt/dagster/dagster_home/
     depends_on:
-      dagster-postgresql:
+      pgbouncer:
         condition: service_healthy
       dagster-pipeline:
         condition: service_started
@@ -221,16 +226,19 @@ services:
     image: ${DAGSTER_DAEMON_IMAGE}
     restart: on-failure
     environment:
-      PGHOST: dagster-postgresql
-      PGUSER: ${DAGSTER_POSTGRES_USER}
-      PGPASSWORD: ${DAGSTER_POSTGRES_PASSWORD}
-      PGDATABASE: ${DAGSTER_POSTGRES_DB}
+      # connect via pgbouncer to improve performance
+      PGHOST: pgbouncer
+      # todo: doesn't work yet, uses default port 5432 instead
+      PGPORT: ${PGBOUNCER_POSTGRES_PORT:?missing/empty $PGBOUNCER_POSTGRES_PORT}
+      PGUSER: ${PGBOUNCER_POSTGRES_USER:?missing/empty $PGBOUNCER_POSTGRES_USER}
+      PGPASSWORD: ${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty $PGBOUNCER_POSTGRES_PASSWORD}
+      PGDATABASE: dagster # determined by pgbouncer!
     volumes: # Make docker client accessible so we can launch containers using host docker
       - /var/run/docker.sock:/var/run/docker.sock
       - /tmp/io_manager_storage:/tmp/io_manager_storage
       - ./etc/dagster:/opt/dagster/dagster_home/
     depends_on:
-      dagster-postgresql:
+      pgbouncer:
         condition: service_healthy
       dagster-pipeline:
         condition: service_started
@@ -253,7 +261,7 @@ services:
       start_period: 10s
       retries: 10
 
-  # pgbouncer acts like a facade in front of PostgreSQL instances (ipl-db, gtfs-db), providing
+  # pgbouncer acts like a facade in front of PostgreSQL instances (ipl-db, gtfs-db, dagster-postgresql), providing
   # - programmatically configurable query routing (used by `make import-new-gtfs`)
   # - improved connect performance for clients, especially Dagster workers
   pgbouncer:
@@ -263,6 +271,8 @@ services:
         condition: service_started
       gtfs-db:
         condition: service_started
+      dagster-postgresql:
+        condition: service_started
     links:
       - ipl-db
       - gtfs-db
@@ -300,7 +310,8 @@ services:
       # see also https://www.postgresql.org/docs/15/libpq-connect.html#id-1.7.3.8.3.5
       # expose ipl-db as `ipl`
       PGBOUNCER_DSN_0: "ipl=host=ipl-db dbname=${IPL_POSTGRES_DB:?missing/empty $IPL_POSTGRES_DB} user=${IPL_POSTGRES_USER:?missing/empty $IPL_POSTGRES_USER} password=${IPL_POSTGRES_PASSWORD:?missing/empty $IPL_POSTGRES_PASSWORD}"
-      # todo: expose dagster-postgresql as `dagster`
+      # expose dagster-postgresql as `dagster`
+      PGBOUNCER_DSN_1: "dagster=host=dagster-postgresql dbname=${DAGSTER_POSTGRES_DB:?missing/empty $DAGSTER_POSTGRES_DB} user=${DAGSTER_POSTGRES_USER:?missing/empty $DAGSTER_POSTGRES_USER} password=${DAGSTER_POSTGRES_PASSWORD:?missing/empty $DAGSTER_POSTGRES_PASSWORD}"
       # expose the latest GTFS import within gtfs-db as `gtfs`
       # $PGBOUNCER_DSN_2 gets generated from /var/pgbouncer-dsn.txt (which is written by gtfs-importer) by the Docker image's entrypoint script.
       # see https://github.com/bitnami/containers/issues/46152#issuecomment-1695320501