fix bug in executor with slowly generated mcps

datahub-project · Oct 31, 2024 · 45cb289 · 45cb289
1 parent b73c43c
commit 45cb289
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/metadata-ingestion/src/datahub/utilities/partition_executor.py b/metadata-ingestion/src/datahub/utilities/partition_executor.py
@@ -294,6 +294,9 @@ def _clearinghouse_worker(self) -> None:  # noqa: C901
         def _handle_batch_completion(
             batch: List[_BatchPartitionWorkItem], future: Future
         ) -> None:
+            nonlocal workers_available
+            workers_available += 1
+
             with clearinghouse_state_lock:
                 for item in batch:
                     keys_no_longer_in_flight.add(item.key)