FIX: Make canonical_stop_sequence use route_pattern_typicality=5 (#205)

Ops Analytics identified an error in the canonical_stop_sequence calculation we were using. This error is triggered by 3 static schedules released in Dec 2023:

    Fall 2023, 2023-12-07T22:35:17+00:00, version D
    Fall 2023, 2023-12-05T18:45:28+00:00, version D
    Fall 2023, 2023-12-04T21:09:26+00:00, version D

When picking which representative_trip_id to use from route_patterns, records with route_pattern_typicality=5 should be used first for any route-pattern/direction combination and then route_pattern_typicality=1 should be used when the first option is not available.

This change will work with static schedule going back to January 2019, but will fail for any older GTFS schedule versions.

This change updates our daily canonical_stop_sequence calculation logic and also deploys a migration for existing canonical_stop_sequence values in the staging RDS.

This change also shrinks our prod migrations into a single file, since no deploy has been pushed to the PROD RDS yet, a single migration file is sufficient.

docker-compose.yml

@@ -37,9 +37,8 @@ services:
     volumes:
       # map credentials to be used by boto3, read-only
       - ~/.aws:/root/.aws:ro
-      # add in july 17 filepahs json that will be the default seed file path
-      - ./python_src/tests/test_files/july_17_filepaths.json:/july_17_filepaths.json
-      - ./python_src/tests/test_files/april_2023_filepaths.json:/april_2023_filepaths.json
+      # add in filepath json that will be the default seed file path
+      - ./python_src/tests/test_files/staging_dec_10.json:/seed_paths.json
     # entrypoint passes in seed file thats added as a volume. if you want to use a different
     # filepath run
     # docker-compose run -v /path/to/files.json:/seed.json seed_metadata --seed-file /seed.json
@@ -49,6 +48,7 @@ services:
         "run",
         "seed_metadata",
         "--clear-static",
+        "--clear-rt",
         "--seed-file",
-        "/april_2023_filepaths.json"
+        "/seed_paths.json"
       ]

python_src/src/lamp_py/migrations/versions/performance_manager_prod/001_5d9a7ee21ae5_initial_prod_schema.py

@@ -157,7 +157,7 @@ def upgrade() -> None:
     op.create_table(
         "static_stop_times",
         sa.Column("pk_id", sa.Integer(), nullable=False),
-        sa.Column("trip_id", sa.String(length=128), nullable=False),
+        sa.Column("trip_id", sa.String(length=512), nullable=False),
         sa.Column("arrival_time", sa.Integer(), nullable=False),
         sa.Column("departure_time", sa.Integer(), nullable=False),
         sa.Column("schedule_travel_time_seconds", sa.Integer(), nullable=True),
@@ -217,7 +217,7 @@ def upgrade() -> None:
         sa.Column("branch_route_id", sa.String(length=60), nullable=True),
         sa.Column("trunk_route_id", sa.String(length=60), nullable=True),
         sa.Column("service_id", sa.String(length=60), nullable=False),
-        sa.Column("trip_id", sa.String(length=128), nullable=False),
+        sa.Column("trip_id", sa.String(length=512), nullable=False),
         sa.Column("direction_id", sa.Boolean(), nullable=True),
         sa.Column("block_id", sa.String(length=128), nullable=True),
         sa.Column("static_version_key", sa.Integer(), nullable=False),
@@ -241,6 +241,12 @@ def upgrade() -> None:
         ["static_version_key", "trunk_route_id"],
         unique=False,
     )
+    op.create_index(
+        "ix_static_trips_composite_4",
+        "static_trips",
+        ["static_version_key", "service_id"],
+        unique=False,
+    )
 
     op.create_table(
         "static_route_patterns",
@@ -249,7 +255,7 @@ def upgrade() -> None:
         sa.Column("direction_id", sa.Boolean(), nullable=True),
         sa.Column("route_pattern_typicality", sa.SmallInteger(), nullable=True),
         sa.Column(
-            "representative_trip_id", sa.String(length=128), nullable=False
+            "representative_trip_id", sa.String(length=512), nullable=False
         ),
         sa.Column("static_version_key", sa.Integer(), nullable=False),
         sa.PrimaryKeyConstraint("pk_id"),
@@ -279,7 +285,7 @@ def upgrade() -> None:
         sa.Column("vp_move_timestamp", sa.Integer(), nullable=True),
         sa.Column("vp_stop_timestamp", sa.Integer(), nullable=True),
         sa.Column("tu_stop_timestamp", sa.Integer(), nullable=True),
-        sa.Column("trip_id", sa.String(length=128), nullable=False),
+        sa.Column("trip_id", sa.String(length=512), nullable=False),
         sa.Column("vehicle_label", sa.String(length=128), nullable=True),
         sa.Column("vehicle_consist", sa.String(), nullable=True),
         sa.Column("static_version_key", sa.Integer(), nullable=False),
@@ -348,12 +354,12 @@ def upgrade() -> None:
         sa.Column("branch_route_id", sa.String(length=60), nullable=True),
         sa.Column("trunk_route_id", sa.String(length=60), nullable=True),
         sa.Column("stop_count", sa.SmallInteger(), nullable=True),
-        sa.Column("trip_id", sa.String(length=128), nullable=False),
+        sa.Column("trip_id", sa.String(length=512), nullable=False),
         sa.Column("vehicle_label", sa.String(length=128), nullable=True),
         sa.Column("vehicle_consist", sa.String(), nullable=True),
         sa.Column("direction", sa.String(length=30), nullable=True),
         sa.Column("direction_destination", sa.String(length=60), nullable=True),
-        sa.Column("static_trip_id_guess", sa.String(length=128), nullable=True),
+        sa.Column("static_trip_id_guess", sa.String(length=512), nullable=True),
         sa.Column("static_start_time", sa.Integer(), nullable=True),
         sa.Column("static_stop_count", sa.SmallInteger(), nullable=True),
         sa.Column("first_last_station_match", sa.Boolean(), nullable=False),
@@ -530,6 +536,7 @@ def downgrade() -> None:
     )
     op.drop_table("static_route_patterns")
 
+    op.drop_index("ix_static_trips_composite_4", table_name="static_trips")
     op.drop_index("ix_static_trips_composite_3", table_name="static_trips")
     op.drop_index("ix_static_trips_composite_2", table_name="static_trips")
     op.drop_index("ix_static_trips_composite_1", table_name="static_trips")