fix failing parquet writes (#181)

Fix type overload on HyperRtRail Job Skip processing of HyperStaticStopTimes because it causes out of memory on ecs.
mbta · Nov 9, 2023 · 02f53af · 02f53af
1 parent b80fb7d
commit 02f53af
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/python_src/src/lamp_py/tableau/jobs/rt_rail.py b/python_src/src/lamp_py/tableau/jobs/rt_rail.py
@@ -126,10 +126,10 @@ def parquet_schema(self) -> pyarrow.schema:
                 ("static_stop_count", pyarrow.int64()),
                 ("first_last_station_match", pyarrow.bool_()),
                 ("static_version_key", pyarrow.int64()),
-                ("travel_time_seconds", pyarrow.int16()),
-                ("dwell_time_seconds", pyarrow.int16()),
-                ("headway_trunk_seconds", pyarrow.int16()),
-                ("headway_branch_seconds", pyarrow.int16()),
+                ("travel_time_seconds", pyarrow.int32()),
+                ("dwell_time_seconds", pyarrow.int32()),
+                ("headway_trunk_seconds", pyarrow.int32()),
+                ("headway_branch_seconds", pyarrow.int32()),
                 ("updated_on", pyarrow.timestamp("us")),
             ]
         )

diff --git a/python_src/src/lamp_py/tableau/pipeline.py b/python_src/src/lamp_py/tableau/pipeline.py
@@ -12,7 +12,7 @@
     HyperStaticFeedInfo,
     HyperStaticRoutes,
     HyperStaticStops,
-    HyperStaticStopTimes,
+    # HyperStaticStopTimes,
     HyperStaticTrips,
 )
 
@@ -27,7 +27,7 @@ def create_hyper_jobs() -> List[HyperJob]:
         HyperStaticFeedInfo(),
         HyperStaticRoutes(),
         HyperStaticStops(),
-        HyperStaticStopTimes(),
+        # HyperStaticStopTimes(),
         HyperStaticTrips(),
     ]