fix(ingest): add lower bound on delta-lake

datahub-project · Oct 16, 2024 · e2d6623 · e2d6623
1 parent d34717f
commit e2d6623
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
@@ -290,9 +290,8 @@
 delta_lake = {
     *s3_base,
     *abs_base,
-    # Version 0.18.0 broken on ARM Macs: https://github.com/delta-io/delta-rs/issues/2577
-    "deltalake>=0.6.3, != 0.6.4, < 0.18.0; platform_system == 'Darwin' and platform_machine == 'arm64'",
-    "deltalake>=0.6.3, != 0.6.4; platform_system != 'Darwin' or platform_machine != 'arm64'",
+    # On deltalake 0.18.x, we were seeing some issues with multiprocessing shutdown.
+    "deltalake>=0.20.2",
 }
 
 powerbi_report_server = {"requests", "requests_ntlm"}