Skip to content

Commit

Permalink
Use a seperate thread pool executor for warehouse pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
Gilbert09 committed Oct 25, 2024
1 parent 76efa0b commit afcba0c
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion posthog/temporal/data_imports/pipelines/pipeline.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Literal
from uuid import UUID
Expand Down Expand Up @@ -253,7 +254,10 @@ def _run(self) -> dict[str, int]:

async def run(self) -> dict[str, int]:
try:
return await asyncio.to_thread(self._run)
# Use a dedicated thread pool to not interfere with the heartbeater thread
with ThreadPoolExecutor(max_workers=5) as pipeline_executor:
loop = asyncio.get_event_loop()
return await loop.run_in_executor(pipeline_executor, self._run)
except PipelineStepFailed as e:
self.logger.exception(f"Data import failed for endpoint with exception {e}", exc_info=e)
raise

0 comments on commit afcba0c

Please sign in to comment.