feat(data-pipelines): allow pinning of hail in requirements

data-pipeline/requirements.txt

@@ -1,5 +1,5 @@
 elasticsearch~=7.17
-hail
+hail==0.2.127
 tqdm
 loguru
 attrs

deploy/deployctl/subcommands/dataproc_cluster.py

@@ -3,6 +3,8 @@
 import subprocess
 import sys
 import typing
+import re
+import importlib.metadata
 
 from deployctl.config import config
 
@@ -24,6 +26,19 @@ def start_cluster(name: str, cluster_args: typing.List[str]) -> None:
         raise RuntimeError("project configuration is required")
 
     with open(os.path.join(DATA_PIPELINE_DIRECTORY, "requirements.txt")) as requirements_file:
+        requirements_hail_version = next(
+            (line.strip().split("==")[1] for line in requirements_file if line.strip().startswith("hail==")), None
+        )
+        local_hail_version = importlib.metadata.version("hail")
+
+        if not requirements_hail_version or not local_hail_version:
+            raise RuntimeError(f"Hail must be both pinned in data-pipeline/requirements.txt, and installed locally")
+        if requirements_hail_version != local_hail_version:
+            raise RuntimeError(
+                f"Local hail version differs from version pinned in data-pipeline/requirements.txt\nRequired version {requirements_hail_version}\nLocal version {local_hail_version}"
+            )
+
+        requirements_file.seek(0)
         requirements = [line.strip() for line in requirements_file.readlines()]
 
     subprocess.check_output(