small bug fixes

IBM · Sep 17, 2024 · 10665b4 · 10665b4
1 parent 9645b17
commit 10665b4
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/...iversal/fdedup_multi_step/kfp_ray/src/fdedup_bucket_processor_compute_execution_params.py b/...iversal/fdedup_multi_step/kfp_ray/src/fdedup_bucket_processor_compute_execution_params.py
@@ -92,7 +92,7 @@ def fdedup_bucket_processor_compute_execution_params(
     )
 
     # Make sure that we have enough memory. We assume that each actor uses 3 GB memory
-    r_mem = 3 * (n_workers + b_processors + fdedup_bucket_processor_num_buckets +
+    r_mem = 2 * (n_workers + b_processors + fdedup_bucket_processor_num_buckets +
                  fdedup_bucket_processor_num_minhashes + fdedup_bucket_processor_num_doc_id)
     if r_mem > cluster_memory:
         print(f"Not enough memory to run de duping, required {r_mem}, available {cluster_memory}")

diff --git a/transforms/universal/fdedup_multi_step/kfp_ray/src/fdedup_filter_compute_execution_params.py b/transforms/universal/fdedup_multi_step/kfp_ray/src/fdedup_filter_compute_execution_params.py
@@ -73,7 +73,7 @@ def fdedup_filter_compute_execution_params(
     print(f"Number of workers: {n_workers}, document actors {fdedup_filter_num_doc_id}")
 
     # Make sure that we have enough memory. We assume that each actor uses 3 GB memory
-    r_mem = 3 * (n_workers + fdedup_filter_num_doc_id)
+    r_mem = 2 * (n_workers + fdedup_filter_num_doc_id)
     if r_mem > cluster_memory:
         print(f"Not enough memory to run de duping, required {r_mem}, available {cluster_memory}")
         print(f"Try to increase the size of the cluster or increase size of the cpu per worker (current {worker_cpu})")

diff --git a/...s/universal/fdedup_multi_step/kfp_ray/src/fdedup_preprocessor_compute_execution_params.py b/...s/universal/fdedup_multi_step/kfp_ray/src/fdedup_preprocessor_compute_execution_params.py
@@ -186,7 +186,7 @@ def _false_negative_probability(ths: float, b: int, r: int) -> float:
     )
 
     # Make sure that we have enough memory. We assume that each actor uses 3 GB memory
-    r_mem = 3 * (n_workers + b_actors + m_actors + d_actors)
+    r_mem = 2 * (n_workers + b_actors + m_actors + d_actors)
     if r_mem > cluster_memory:
         print(f"Not enough memory to run de duping, required {r_mem}, available {cluster_memory}")
         print(f"Try to increase the size of the cluster or increase size of the cpu per worker (current {worker_cpu})")