fix(ingress): make certbot dependencies measured

custom-domain/dstack-ingress/Dockerfile

@@ -58,6 +58,21 @@ RUN mkdir -p \
     /var/lib/haproxy \
     /evidences
 
+# Install the complete Python runtime during the image build.  Runtime
+# installation from PyPI would leave certbot (which handles TLS private keys)
+# outside the measured image.  Keep versions explicit; update this list only
+# together with a new image build and digest.
+COPY requirements.txt /tmp/requirements.txt
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONHASHSEED=0 \
+    SOURCE_DATE_EPOCH=0
+RUN python3 -m venv /opt/app-venv && \
+    /opt/app-venv/bin/pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt && \
+    rm -rf /root/.cache/pip /tmp/requirements.txt && \
+    # Normalize files created by wheel installers for reproducible layers.
+    find /opt/app-venv -type f -exec touch -d @0 {} + && \
+    find /opt/app-venv -type d -exec touch -d @0 {} +
+
 # Install scripts with deterministic permissions via bind mount
 RUN --mount=type=bind,source=scripts,target=/tmp/scripts,ro \
     /bin/bash -o pipefail -c 'set -euo pipefail; \

custom-domain/dstack-ingress/README.md

@@ -448,3 +448,10 @@ and renewal therefore never interrupt serving traffic. In this mode haproxy
 starts on a self-signed placeholder certificate — it has to be listening before
 the first certificate can be issued — and reloads onto the real one as soon as
 it arrives.
+
+### Deterministic TLS keys (optional)
+
+Set `DETERMINISTIC_TLS_KEY=true` to derive a stable P-256 TLS key per domain from the
+internal dstack agent `/GetKey` endpoint. The default is `false`; when unset, Certbot
+retains its normal key generation behavior. The derived key is used only to generate the
+CSR and is not represented by the agent's signature chain.

custom-domain/dstack-ingress/requirements.in

@@ -0,0 +1,8 @@
+certbot==5.8.0
+requests==2.32.3
+boto3==1.35.99
+botocore==1.35.99
+certbot-dns-cloudflare==4.0.0
+certbot-dns-route53==5.1.0
+certbot-dns-linode==1.4.0
+certbot-dns-namecheap==1.0.0

custom-domain/dstack-ingress/requirements.txt

@@ -0,0 +1,95 @@
+# Generated from /tmp/wheels; all artifacts are hash-pinned.
+PyNamecheap==0.0.3 \
+    --hash=sha256:d428f9455f6226f5dbe2e67c5a81b5b37a215edf94d55f7576164904c4e76b72
+acme==5.8.0 \
+    --hash=sha256:637501767248156545d85c23b806381ac346c5eac759f8e35052af85b7bb3933
+attrs==26.1.0 \
+    --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309
+beautifulsoup4==4.15.0 \
+    --hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9
+boto3==1.35.99 \
+    --hash=sha256:83e560faaec38a956dfb3d62e05e1703ee50432b45b788c09e25107c5058bd71
+botocore==1.35.99 \
+    --hash=sha256:b22d27b6b617fc2d7342090d6129000af2efd20174215948c0d7ae2da0fab445
+certbot==5.8.0 \
+    --hash=sha256:c06793e6a0169b07ee09e11e6a017c7d5a77310055d693fffd66c53363ab87ff
+certbot-dns-namecheap==1.0.0 \
+    --hash=sha256:0801da6093f517720e86efbc7f42515a14ec135fe7e5bce118e8d0b6e6fb51e5
+certbot-dns-cloudflare==4.0.0 \
+    --hash=sha256:b4ab718b9ca946fc4558dcb7b79c3ef0a12ab049fdeb29bccbd1fafa48aebff0
+certbot-dns-linode==1.4.0 \
+    --hash=sha256:a161a2f95cd2d11b23abb18fa96a35448b9dc144072fe580035e33b362cc8064
+certbot-dns-route53==5.1.0 \
+    --hash=sha256:64c0594b28cc8544559f5981cd3ea514c09f0911c54d045363573d2146247eba
+certifi==2026.7.22 \
+    --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775
+cffi==2.1.1 \
+    --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \
+    --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632
+charset-normalizer==3.5.1 \
+    --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \
+    --hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8
+cloudflare==2.19.4 \
+    --hash=sha256:3b6000a01a237c23bccfdf6d20256ea5111ec74a826ae9e74f9f0e5bb5b2383f
+ConfigArgParse==1.7.5 \
+    --hash=sha256:1e63fdffedf94da9cd435fc13a1cd24777e76879dd2343912c1f871d4ac8c592
+configobj==5.0.9 \
+    --hash=sha256:1ba10c5b6ee16229c79a05047aeda2b55eb4e80d7c7d8ecf17ec1ca600c79882
+cryptography==50.0.1 \
+    --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a
+distro==1.9.0 \
+    --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2
+dns-lexicon==3.25.2 \
+    --hash=sha256:0592a30d8d721226668b34b736d7af1332d5c5fccf138626964d644c2b17b9be
+dnspython==2.8.0 \
+    --hash=sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af
+filelock==3.32.5 \
+    --hash=sha256:142cd9fa77a872c5e78c62329a0d15278fadc686eb89e760017968961a4fd6b2
+idna==3.19 \
+    --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4
+jmespath==1.1.0 \
+    --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64
+josepy==2.2.0 \
+    --hash=sha256:63e9dd116d4078778c25ca88f880cc5d95f1cab0099bebe3a34c2e299f65d10b
+jsonlines==4.0.0 \
+    --hash=sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55
+mock==5.2.0 \
+    --hash=sha256:7ba87f72ca0e915175596069dbbcc7c75af7b5e9b9bc107ad6349ede0819982f
+parsedatetime==2.6 \
+    --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b
+pycparser==3.0 \
+    --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992
+pyOpenSSL==26.4.0 \
+    --hash=sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c
+PyOTP==2.10.0 \
+    --hash=sha256:1df2f6a1bcc3bb0716172a5215ddc2f8c7c7fd26a13df9927d52e1746934836c
+pyRFC3339==2.1.0 \
+    --hash=sha256:560f3f972e339f579513fe1396974352fd575ef27caff160a38b312252fcddf3
+python-dateutil==2.9.0.post0 \
+    --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
+PyYAML==6.0.3 \
+    --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \
+    --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d
+requests==2.32.3 \
+    --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
+requests-file==3.0.1 \
+    --hash=sha256:d0f5eb94353986d998f80ac63c7f146a307728be051d4d1cd390dbdb59c10fa2
+requests-unixsocket==0.4.1 \
+    --hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d
+s3transfer==0.10.4 \
+    --hash=sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e
+setuptools==84.0.0 \
+    --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670
+six==1.17.0 \
+    --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274
+soupsieve==2.9.2 \
+    --hash=sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823
+tldextract==5.3.2 \
+    --hash=sha256:6c90d2a259f5c89f4fcf01f97af15708416a59ffedddff006d67222ab30d0fb0
+typing_extensions==4.16.0 \
+    --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8
+urllib3==2.7.0 \
+    --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897
+zope.interface==8.6 \
+    --hash=sha256:9217b1123f6aeec9ddf1789bffd83da3123546d551c164a99f862a5d1f5ac0f8 \
+    --hash=sha256:a43e669d68fd8c10fe315812f7e1d262c6c00e9667f29f799a3771f9a3b5b41d

custom-domain/dstack-ingress/scripts/certman.py

@@ -6,7 +6,7 @@
 import re
 import subprocess
 import sys
-import pkg_resources
+from importlib import metadata
 from typing import List, Optional, Tuple
 
 # Add script directory to path to import dns_providers
@@ -37,143 +37,26 @@ def _detect_provider_type(self) -> str:
         """Detect provider type (reuse factory logic)."""
         return DNSProviderFactory._detect_provider_type()
 
+    @staticmethod
+    def _cert_name(domain: str) -> str:
+        """Match certbot's stable lineage name for a domain."""
+        return domain.lstrip("*.").replace("*", "wildcard-")
+
     def install_plugin(self) -> bool:
         """Install certbot plugin for the current provider."""
         if not self.provider.CERTBOT_PACKAGE:
             print(f"No certbot package defined for {self.provider_type}")
             return False
 
-        # First ensure certbot is installed in the current environment
-        self._ensure_certbot_in_env()
-
-        # Check if plugin is already installed
         try:
+            self._ensure_certbot_in_env()
             __import__(self.provider.CERTBOT_PLUGIN_MODULE)
-            print(
-                f"Plugin {self.provider.CERTBOT_PACKAGE} is already installed")
+            print(f"Plugin {self.provider.CERTBOT_PACKAGE} is available")
             return True
-        except ImportError:
-            pass  # Plugin not installed, continue with installation
-
-        print(f"Installing certbot plugin: {self.provider.CERTBOT_PACKAGE}")
-
-        # Try multiple installation methods
-        install_methods = []
-
-        # Method 1: Use the same python executable that's running this script
-        install_methods.append(
-            [sys.executable, "-m", "pip", "install", self.provider.CERTBOT_PACKAGE])
-
-        # Method 2: Use virtual environment pip if available
-        if "VIRTUAL_ENV" in os.environ:
-            venv_pip = os.path.join(os.environ["VIRTUAL_ENV"], "bin", "pip")
-            if os.path.exists(venv_pip):
-                install_methods.append(
-                    [venv_pip, "install", self.provider.CERTBOT_PACKAGE])
-
-        # Method 3: Use system pip
-        install_methods.append(
-            ["pip", "install", self.provider.CERTBOT_PACKAGE])
-
-        # Method 4: Use pip3
-        install_methods.append(
-            ["pip3", "install", self.provider.CERTBOT_PACKAGE])
-
-        success = False
-        for i, pip_cmd in enumerate(install_methods):
-            print(f"Trying installation method {i+1}")
-            print(f"Command: {' '.join(pip_cmd)}")
-            try:
-                result = subprocess.run(
-                    pip_cmd, capture_output=True, text=True)
-                if result.returncode == 0:
-                    print(f"Installation method {i+1} succeeded")
-                    success = True
-                    break
-                else:
-                    print(f"Installation method {i+1} failed: {result.stderr}")
-            except Exception as e:
-                print(f"Installation method {i+1} exception: {e}")
-
-        if not success:
-            print(f"All installation methods failed", file=sys.stderr)
+        except (ImportError, RuntimeError) as exc:
+            print(f"Required certbot dependency is missing from the measured image: {exc}", file=sys.stderr)
             return False
 
-        print(f"Successfully installed {self.provider.CERTBOT_PACKAGE}")
-
-        # Diagnostic information for troubleshooting
-        try:
-            print(f"Installed to Python: {sys.executable}")
-
-            # Show certbot command
-            certbot_cmd = self._get_certbot_command()
-            print(f"Using certbot: {' '.join(certbot_cmd)}")
-
-            try:
-                dist = pkg_resources.get_distribution(
-                    self.provider.CERTBOT_PACKAGE)
-                print(f"Package version: {dist.version} at {dist.location}")
-            except pkg_resources.DistributionNotFound:
-                print("Warning: Package not found in current environment")
-        except Exception as diag_error:
-            print(f"Diagnostic error: {diag_error}")
-
-        # Verify plugin installation
-        try:
-            __import__(self.provider.CERTBOT_PLUGIN_MODULE)
-            print(
-                f"Plugin {self.provider.CERTBOT_PLUGIN} successfully imported")
-
-            # Test if plugin is recognized by certbot
-            certbot_cmd = self._get_certbot_command()
-            test_cmd = certbot_cmd + ["plugins"]
-            test_result = subprocess.run(
-                test_cmd, capture_output=True, text=True, timeout=10)
-
-            if test_result.returncode == 0 and self.provider.CERTBOT_PLUGIN in test_result.stdout:
-                print(
-                    f"✓ Plugin {self.provider.CERTBOT_PLUGIN} is available in certbot")
-                return True
-            else:
-                print(
-                    f"Warning: {self.provider.CERTBOT_PLUGIN} plugin not found in certbot plugins list")
-                if test_result.stderr:
-                    print(f"Plugin test stderr: {test_result.stderr}")
-
-                # Debug plugin registration
-                self._debug_plugin_registration()
-
-                # Try force reinstall to fix plugin registration
-                print("Attempting to fix plugin registration...")
-                try:
-                    force_cmd = [sys.executable, "-m", "pip", "install", "--force-reinstall",
-                                 "--no-deps", self.provider.CERTBOT_PACKAGE]
-                    print(f"Running: {' '.join(force_cmd)}")
-                    force_result = subprocess.run(
-                        force_cmd, capture_output=True, text=True)
-
-                    if force_result.returncode == 0:
-                        # Test again after reinstall
-                        retest_cmd = certbot_cmd + ["plugins"]
-                        retest_result = subprocess.run(
-                            retest_cmd, capture_output=True, text=True, timeout=10)
-                        if retest_result.returncode == 0 and self.provider.CERTBOT_PLUGIN in retest_result.stdout:
-                            print(f"✓ Plugin registration fixed after reinstall")
-                            return True
-                        else:
-                            print(f"Plugin still not registered, may work anyway")
-                    else:
-                        print(f"Force reinstall failed: {force_result.stderr}")
-                except Exception as fix_error:
-                    print(f"Plugin fix attempt failed: {fix_error}")
-
-                # Continue anyway - may work in Docker environments
-                return True
-
-        except Exception as e:
-            print(f"Plugin verification warning: {e}")
-            return True
-
     def _ensure_certbot_in_env(self) -> None:
         """Ensure certbot is installed in the current Python environment."""
 
@@ -182,24 +65,8 @@ def _ensure_certbot_in_env(self) -> None:
             import certbot
             print(f"✓ Certbot module available in current environment")
             return
-        except ImportError:
-            print(f"Certbot module not found, installing...")
-
-        # Install certbot if not available
-        try:
-            install_cmd = [sys.executable, "-m", "pip", "install", "certbot"]
-            print(f"Running: {' '.join(install_cmd)}")
-            result = subprocess.run(
-                install_cmd, capture_output=True, text=True)
-
-            if result.returncode == 0:
-                print(f"✓ Certbot installed successfully in current environment")
-            else:
-                print(f"Failed to install certbot: {result.stderr}")
-                # Continue anyway - may still work
-        except Exception as e:
-            print(f"Error installing certbot: {e}")
-            # Continue anyway - may still work
+        except ImportError as exc:
+            raise RuntimeError("certbot is missing from the measured image") from exc
 
     def _get_certbot_command(self) -> List[str]:
         """Get the correct certbot command that uses the same Python environment."""
@@ -223,7 +90,6 @@ def _get_certbot_command(self) -> List[str]:
     def _debug_plugin_registration(self) -> None:
         """Debug why plugin is not being registered by certbot."""
         try:
-            import pkg_resources
             print("=== Plugin Registration Debug ===")
 
             # Show which certbot we're using
@@ -232,11 +98,10 @@ def _debug_plugin_registration(self) -> None:
 
             # Check entry points
             try:
-                entry_points = list(
-                    pkg_resources.iter_entry_points('certbot.plugins'))
+                entry_points = list(metadata.entry_points(group='certbot.plugins'))
                 print(f"Found {len(entry_points)} certbot plugins:")
                 for ep in entry_points:
-                    print(f"  - {ep.name}: {ep.module_name}")
+                    print(f"  - {ep.name}: {ep.value}")
 
                 # Look specifically for our plugin
                 plugin_eps = [ep for ep in entry_points if ep.name ==
@@ -278,6 +143,11 @@ def setup_credentials(self) -> bool:
     def _build_certbot_command(self, action: str, domain: str, email: str) -> List[str]:
         """Build certbot command using provider configuration."""
         certbot_cmd = self._get_certbot_command()
+        deterministic = os.environ.get("DETERMINISTIC_TLS_KEY", "false").lower() == "true"
+        if deterministic and action == "certonly":
+            key_path = f"/etc/letsencrypt/live/{self._cert_name(domain)}/privkey.pem"
+            csr_path = f"/etc/letsencrypt/csr/{self._cert_name(domain)}.csr"
+            subprocess.run(["python3", "/scripts/deterministic_key.py", domain, key_path, csr_path], check=True)
 
         # Challenge-delegation mode: when DELEGATION_ZONE is set, answer the
         # DNS-01 challenge in a delegated zone via a manual hook instead of the
@@ -330,6 +200,8 @@ def _build_certbot_command(self, action: str, domain: str, email: str) -> List[s
                     f"Credentials file does not exist: {credentials_file}")
 
         if action == "certonly":
+            if deterministic:
+                base_cmd.extend(["--csr", csr_path, "--cert-path", f"/etc/letsencrypt/live/{self._cert_name(domain)}/cert.pem", "--fullchain-path", f"/etc/letsencrypt/live/{self._cert_name(domain)}/fullchain.pem", "--chain-path", f"/etc/letsencrypt/live/{self._cert_name(domain)}/chain.pem"])
             base_cmd.extend(["--agree-tos", "--no-eff-email"])
             # The ACME contact address is optional (RFC 8555 section 7.3), and
             # it is published: the account document is served as attestation
@@ -344,6 +216,11 @@ def _build_certbot_command(self, action: str, domain: str, email: str) -> List[s
             base_cmd.extend(["-d", domain])
         if staging_enabled():
             base_cmd.extend(["--staging"])
+        # Allow local ACME test servers (for example Pebble) without changing
+        # the production/staging defaults.
+        acme_server = os.environ.get("ACME_DIRECTORY_URL", "").strip()
+        if acme_server:
+            base_cmd.extend(["--server", acme_server])
 
         if getattr(self.provider, 'CERTBOT_PROPAGATION_SECONDS'):
             propagation_seconds = self.provider.CERTBOT_PROPAGATION_SECONDS