diff --git a/deploy/deployer/ci-fleet-deployer-cleanup.service b/deploy/deployer/ci-fleet-deployer-cleanup.service new file mode 100644 index 0000000..e1c44f0 --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-cleanup.service @@ -0,0 +1,19 @@ +[Unit] +Description=Run application-owned scoped deployer cleanup +After=docker.service +Wants=docker.service + +[Service] +Type=oneshot +User=root +ExecStart=/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh cleanup +Environment=CI_FLEET_DEPLOYER_CONFIG=/var/lib/ci-fleet-deployer/active-policy.conf +TimeoutStartSec=15min +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadOnlyPaths=/etc/ci-fleet-deployer /opt/ci-fleet-deployer +ReadWritePaths=/var/lib/ci-fleet-deployer /var/log/ci-fleet-deployer /var/lock/ci-fleet-deployer /run/docker.sock +RestrictSUIDSGID=yes diff --git a/deploy/deployer/ci-fleet-deployer-cleanup.timer b/deploy/deployer/ci-fleet-deployer-cleanup.timer new file mode 100644 index 0000000..0ac89aa --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-cleanup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run scoped ci-fleet deployer cleanup daily + +[Timer] +OnCalendar=daily +RandomizedDelaySec=15min +Persistent=true +Unit=ci-fleet-deployer-cleanup.service + +[Install] +WantedBy=timers.target diff --git a/deploy/deployer/ci-fleet-deployer-drain.service b/deploy/deployer/ci-fleet-deployer-drain.service new file mode 100644 index 0000000..eac308a --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-drain.service @@ -0,0 +1,17 @@ +[Unit] +Description=Drain the ci-fleet deployer before maintenance + +[Service] +Type=oneshot +User=root +ExecStart=/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh drain +Environment=CI_FLEET_DEPLOYER_CONFIG=/var/lib/ci-fleet-deployer/active-policy.conf +TimeoutStartSec=2min +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadOnlyPaths=/etc/ci-fleet-deployer /opt/ci-fleet-deployer +ReadWritePaths=/var/lib/ci-fleet-deployer /var/log/ci-fleet-deployer /var/lock/ci-fleet-deployer +RestrictSUIDSGID=yes diff --git a/deploy/deployer/ci-fleet-deployer-health.service b/deploy/deployer/ci-fleet-deployer-health.service new file mode 100644 index 0000000..2cb01b5 --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-health.service @@ -0,0 +1,19 @@ +[Unit] +Description=Check ci-fleet deployer health +After=docker.service +Wants=docker.service + +[Service] +Type=oneshot +User=root +ExecStart=/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh health +Environment=CI_FLEET_DEPLOYER_CONFIG=/var/lib/ci-fleet-deployer/active-policy.conf +TimeoutStartSec=2min +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadOnlyPaths=/etc/ci-fleet-deployer /opt/ci-fleet-deployer +ReadWritePaths=/var/lib/ci-fleet-deployer /var/log/ci-fleet-deployer /var/lock/ci-fleet-deployer /run/docker.sock +RestrictSUIDSGID=yes diff --git a/deploy/deployer/ci-fleet-deployer-health.timer b/deploy/deployer/ci-fleet-deployer-health.timer new file mode 100644 index 0000000..637cfb9 --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-health.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Check ci-fleet deployer health every five minutes + +[Timer] +OnActiveSec=2min +OnUnitActiveSec=5min +RandomizedDelaySec=30s +Persistent=true +Unit=ci-fleet-deployer-health.service + +[Install] +WantedBy=timers.target diff --git a/deploy/deployer/ci-fleet-deployer.service b/deploy/deployer/ci-fleet-deployer.service new file mode 100644 index 0000000..ebefc9e --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer.service @@ -0,0 +1,25 @@ +[Unit] +Description=Run one approved ci-fleet deployment +After=docker.service network-online.target +Wants=docker.service network-online.target + +[Service] +Type=oneshot +User=root +Group=root +ExecStart=/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh deploy +Environment=CI_FLEET_DEPLOYER_CONFIG=/var/lib/ci-fleet-deployer/active-policy.conf +Environment=CI_FLEET_DEPLOYER_REQUEST=/var/lib/ci-fleet-deployer/request.conf +TimeoutStartSec=45min +# A hung adapter must not hold the lock or inhibitor beyond the documented +# 45-minute deployment bound; stop quickly after TERM and let KILL follow. +TimeoutStopSec=30s +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadOnlyPaths=/etc/ci-fleet-deployer /opt/ci-fleet-deployer +ReadWritePaths=/var/lib/ci-fleet-deployer /var/log/ci-fleet-deployer /var/lock/ci-fleet-deployer /run/docker.sock +RestrictSUIDSGID=yes +LockPersonality=yes diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md new file mode 100644 index 0000000..6b2045a --- /dev/null +++ b/docs/DEPLOYER-HOST.md @@ -0,0 +1,329 @@ +# Deployer host installation and operation + +This runbook installs the generic ci-fleet deployer runtime on a dedicated Linux host. It does not deploy an application by itself. Application repositories retain their own deployment logic through the narrow adapter contract below. + +A deployer is a separate trust boundary. It must never register as an ordinary GitHub Actions runner, accept pull-request jobs, share a Docker daemon with CI workers, or run on the ordinary controller host. Staging and production require different hosts or equally isolated security boundaries, identities, credential scopes, state, approval evidence, and target policy. + +## Capability and security model + +The runtime accepts only a full source commit, an image reference ending in `@sha256:<64 lowercase hex>`, exact matching approval evidence, explicit environment and target identities, a protected credential reference, and checkpoint evidence. It never follows `latest`, a branch, a moving tag, or an implicit production default. + +GitHub-native Environment protection is optional, not assumed. In particular, do not assume a private repository on GitHub Free has protected Environments, Environment secrets, branch protection, or rulesets. Use one of these fail-closed approval providers: + +- `manual-exact-head`: a protected host-local evidence file records an approval outside the secret store; +- `external-exact-head`: an external policy system writes the same bounded evidence contract; +- `github-environment`: accepted only with separate host-local capability evidence proving exact-head Environment protection was verified for this installation. + +A secret store proves only that a credential exists. It is not approval evidence. Every provider still binds the environment, target, source commit, artifact digest, approving identity, policy identity, approval ID, and UTC time. + +Production remains separately gated during the controlled-migration phase. `ENVIRONMENT=production` is rejected unless `PRODUCTION_AUTHORIZATION_EVIDENCE_PATH` names an additional protected exact-target, exact-head, exact-artifact authorization record. Ordinary approval or credential access does not satisfy this gate, and the field is rejected outside production. + +The systemd services run as root because access to the Docker socket is root-equivalent and protected credential references may be root-only. `NoNewPrivileges`, a read-only host filesystem, explicit writable paths, private temporary storage, and no supplementary service account reduce accidental reach, but they do not turn Docker access into a low-privilege boundary. Put nothing else on this host. + +## Supported host and prerequisites + +The installer supports Debian 12/13 and Ubuntu 22.04/24.04 with systemd. It checks Docker Engine, synchronized time, disk capacity, DNS, HTTPS reachability, and Compose v2 when `REQUIRE_COMPOSE=1`. The configured network host is supplied to curl over standard input so a private endpoint is not placed in the process argument list. + +It rejects: + +- ordinary ci-fleet controller state or runner units; +- any unrelated running or stopped Docker container; +- any unrelated custom Docker network or volume; +- unsafe owners, modes, symlinks, traversal, malformed configuration, or ambiguous identities. + +`--check` is read-only: it creates no user, directory, lock, release, service, timer, image, container, registration, or GitHub state. Mutating modes require root. + +## Filesystem contract + +| Path | Owner/mode | Purpose | Uninstall | +| --- | --- | --- | --- | +| `/etc/ci-fleet-deployer/` | root `0700` | host-local policy boundary | retained | +| `/etc/ci-fleet-deployer/deployer.conf` | root `0600` | bounded non-secret policy and references | retained | +| `/etc/ci-fleet-deployer/adapters/` | root `0700` | application-owned adapter | retained | +| `/etc/ci-fleet-deployer/credentials/` | root `0700` | credential files | retained | +| `/etc/ci-fleet-deployer/evidence/` | root `0700` | approval/capability/checkpoint evidence | retained | +| `/opt/ci-fleet-deployer/releases//` | root `0755` | immutable core runtime release | retained | +| `/opt/ci-fleet-deployer/current` | root symlink | atomically selected core release | removed | +| `/var/lib/ci-fleet-deployer/` | root `0700` | active policy, request, drain, transaction, LKG state | retained except transient drain/active state | +| `/var/lock/ci-fleet-deployer/` | root `0700` | flock serialization boundary | retained | +| `/var/log/ci-fleet-deployer/` | root `0700` | secret-free audit log | retained | +| `/etc/systemd/system/ci-fleet-deployer*` | root `0644` | deploy, health, cleanup, and drain units/timers | removed | + +Releases, last-known-good state, audit records, configuration, credentials, and evidence are retained deliberately. Their retention or destruction is a separate operator decision. + +## Application adapter contract + +The adapter is an application-owned root-only executable under `/etc/ci-fleet-deployer/adapters`. Its SHA-256 is pinned in `deployer.conf`. Core invokes exactly one positional operation: + +```text +adapter validate +adapter health +adapter cleanup +adapter deploy +adapter rollback +``` + +`validate` must be non-mutating and must prove that the candidate policy is usable before core changes the active release. `health` returns zero only when the deployer and application-owned contract are healthy. `cleanup` may remove only resources carrying the application's exact deployer ownership identity; it must never run global prune or touch unrelated resources. `deploy` reads the active policy and request paths from the documented environment variables and owns application-specific staging, rollout, health, and rollback. `rollback` restores application state compatible with the recorded last-known-good core policy. + +Every Docker resource the adapter creates (containers, custom networks, volumes) must carry the labels `io.randomdevelopment.ci-fleet.role=deployer` and `io.randomdevelopment.ci-fleet.identity=` from the active policy. Host-role isolation rejects any unlabelled or differently labelled workload as unrelated, so unlabelled adapter resources break subsequent health, cleanup, deployment, and installer operations. + +Operations have no interactive input. Zero means success; nonzero means failure. Direct installer validation and health calls are limited to two minutes; rollback is limited to 45 minutes. The adapter must avoid child processes that outlive those bounds, redact logs, and never print credential contents, authorization headers, cookies, private endpoints, or secret-manager responses. Core validates immutable identifiers and approval evidence; it cannot validate application-specific correctness. + +Rollback must be atomic from the adapter's perspective: nonzero restores the pre-call application state; zero means rollback health is already verified. Rollback is exposed only through the transactional installer, never as a direct runtime operation. For rollback only, core exports `CI_FLEET_DEPLOYER_ROLLBACK_COMMIT`; the adapter atomically creates that root-owned mode-`0600` file as its final successful step. Core stages and selects the last-known-good core before invoking the adapter, restores the newer core on an uncommitted failure, and consumes committed last-known-good state only after core/application alignment. A committed rollback interrupted after the adapter returns is finalized from the retained transaction on the next serialized installer operation. + +## Prepare host-local files + +All commands in this section run on the dedicated deployer host and change host-local state. They do not contact GitHub or deploy an application. + +```bash +sudo install -d -o root -g root -m 0700 \ + /etc/ci-fleet-deployer \ + /etc/ci-fleet-deployer/adapters \ + /etc/ci-fleet-deployer/credentials \ + /etc/ci-fleet-deployer/evidence +adapter_sha=$(sha256sum ./application-adapter | cut -d' ' -f1) +sudo install -o root -g root -m 0700 ./application-adapter \ + "/etc/ci-fleet-deployer/adapters/application-adapter.$adapter_sha" +sudo install -o root -g root -m 0600 /dev/null \ + /etc/ci-fleet-deployer/credentials/application.credential +sudoedit /etc/ci-fleet-deployer/credentials/application.credential +``` + +`sudoedit` writes the credential without displaying it. Do not use a secret value in a command argument, shell history, environment variable, fixture, or log. An approved external secret-manager reference may replace the regular file; the adapter retrieves the value without core seeing it. + +Create checkpoint evidence only after an operator or approved external backup adapter has produced a recoverable checkpoint. Core cannot see or operate a hypervisor and does not claim to verify one. This example records evidence, not a secret: + +```bash +sudo install -o root -g root -m 0600 /dev/null \ + /etc/ci-fleet-deployer/evidence/checkpoint.conf +sudoedit /etc/ci-fleet-deployer/evidence/checkpoint.conf +``` + +Fictional checkpoint evidence: + +```text +SCHEMA_VERSION=1 +ENVIRONMENT=staging +TARGET_ID=example-staging +CHECKPOINT_ID=checkpoint-20260808-1 +RECORDED_AT=2026-08-08T19:55:00Z +``` + +Create independent exact-head approval evidence: + +```bash +sudo install -o root -g root -m 0600 /dev/null \ + /etc/ci-fleet-deployer/evidence/approval.conf +sudoedit /etc/ci-fleet-deployer/evidence/approval.conf +``` + +Fictional manual/external evidence: + +```text +SCHEMA_VERSION=1 +ENVIRONMENT=staging +TARGET_ID=example-staging +SOURCE_COMMIT=1111111111111111111111111111111111111111 +ARTIFACT_IMAGE=registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +APPROVAL_IDENTITY=example-reviewer +POLICY_IDENTITY=example-staging-policy-v1 +APPROVAL_ID=approval-20260808-1 +APPROVED_AT=2026-08-08T20:00:00Z +``` + +For `github-environment`, also create mode-`0600` capability evidence. This file must be produced by an authorized capability check; copying the fictional text is not proof: + +```text +SCHEMA_VERSION=1 +ENVIRONMENT=staging +TARGET_ID=example-staging +ENVIRONMENT_PROTECTION=verified +EXACT_HEAD=1111111111111111111111111111111111111111 +CAPABILITY_ID=example-capability-check +CHECKED_AT=2026-08-08T20:00:00Z +``` + +The `ENVIRONMENT` and `TARGET_ID` bind the evidence to one installation; capability evidence produced for a different installation is rejected even when the source commit matches. + +For production, a separate authorized process must also create evidence such as: + +```text +SCHEMA_VERSION=1 +ENVIRONMENT=production +TARGET_ID=example-production +SOURCE_COMMIT=1111111111111111111111111111111111111111 +ARTIFACT_IMAGE=registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +AUTHORIZED_BY=example-production-authorizer +GATE_ID=production-gate-20260808-1 +AUTHORIZED_AT=2026-08-08T20:00:00Z +``` + +Production deployment is currently disabled: `docs/DESIGN-DECISIONS.md` keeps production deployment paths separately gated, and the runtime rejects every production request until an accepted decision enables them. The evidence above documents the intended record shape only; it does not lift the gate. + +Create the bounded configuration. Values cannot contain shell expressions; the installer parses `KEY=VALUE` without sourcing it. Unknown, duplicate, empty, malformed, or missing keys fail closed. + +```bash +core_ref=$(git rev-parse HEAD) +adapter_sha=$(sha256sum ./application-adapter | cut -d' ' -f1) +adapter_path=/etc/ci-fleet-deployer/adapters/application-adapter.$adapter_sha +sudo install -o root -g root -m 0600 /dev/null \ + /etc/ci-fleet-deployer/deployer.conf +{ + printf 'SCHEMA_VERSION=1\n' + printf 'CORE_REF=%s\n' "$core_ref" + printf 'ENVIRONMENT=staging\n' + printf 'TARGET_ID=example-staging\n' + printf 'DEPLOYER_IDENTITY=staging-deployer-01\n' + printf 'ADAPTER_PATH=%s\n' "$adapter_path" + printf 'ADAPTER_SHA256=%s\n' "$adapter_sha" + printf 'CREDENTIAL_PROVIDER=file\n' + printf 'CREDENTIAL_REF=/etc/ci-fleet-deployer/credentials/application.credential\n' + printf 'CREDENTIAL_SCOPE=staging\n' + printf 'APPROVAL_PROVIDER=manual-exact-head\n' + printf 'APPROVAL_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/approval.conf\n' + printf 'CHECKPOINT_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/checkpoint.conf\n' + printf 'SOURCE_COMMIT=1111111111111111111111111111111111111111\n' + printf 'ARTIFACT_IMAGE=registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n' + printf 'NETWORK_HOST=registry.example.invalid\n' + printf 'MIN_DISK_GIB=10\n' + printf 'REQUIRE_COMPOSE=1\n' +} | sudo tee /etc/ci-fleet-deployer/deployer.conf >/dev/null +sudo chmod 0600 /etc/ci-fleet-deployer/deployer.conf +``` + +For an external secret manager, use `CREDENTIAL_PROVIDER=external` and a non-secret reference shaped like `external:example-vault:staging/deployer`. For GitHub Environment approval, use `APPROVAL_PROVIDER=github-environment` and add `APPROVAL_CAPABILITY_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/github-capability.conf`. Production additionally requires `PRODUCTION_AUTHORIZATION_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/production-authorization.conf`. + +## Install, check, repair, upgrade, drain, and rollback + +These commands run from a clean checkout at the exact `CORE_REF` on the dedicated deployer host. + +Fresh install changes the host: + +```bash +sudo ./scripts/install-deployer.sh \ + --install --config /etc/ci-fleet-deployer/deployer.conf +``` + +Read-only validation: + +```bash +sudo ./scripts/install-deployer.sh \ + --check --config /etc/ci-fleet-deployer/deployer.conf +``` + +Repair owned drift without selecting a different environment or target; changes the host only when drift exists: + +```bash +sudo ./scripts/install-deployer.sh \ + --repair --config /etc/ci-fleet-deployer/deployer.conf +``` + +Upgrade after updating the exact core/source/artifact/approval/checkpoint fields; changes the host transactionally. Adapter bytes are immutable too: install changed bytes at a new digest-suffixed path, update `ADAPTER_PATH` and `ADAPTER_SHA256` together, and retain the old path while any deployed or last-known-good policy names it. Never overwrite the adapter at its existing path; pre-convergence verification intentionally requires those old bytes so rollback remains executable. + +```bash +sudo ./scripts/install-deployer.sh \ + --upgrade --config /etc/ci-fleet-deployer/deployer.conf +``` + +The adapter candidate passes `validate` before activation. Core stages a complete commit-pinned release, atomically switches it, writes active policy/state with mode `0600`, and derives last-known-good from the atomically published deployed policy/state snapshot rather than an undeployed candidate. A failed candidate does not replace the current release or remove drain state. + +Drain before reboot or maintenance; changes drain state and refuses while a deployment is active: + +```bash +sudo ./scripts/install-deployer.sh \ + --drain --config /etc/ci-fleet-deployer/deployer.conf +sudo systemctl start ci-fleet-deployer-drain.service +``` + +After maintenance, explicitly resume through the same serialized installer boundary. This removes the managed drain marker only after full installed-state convergence, role isolation, and active-policy health pass; it is idempotent: + +```bash +sudo ./scripts/install-deployer.sh \ + --resume --config /etc/ci-fleet-deployer/deployer.conf +``` + +Rollback changes the active core/application state but deliberately does not overwrite or depend on the operator-owned candidate evidence or registry preflight; it uses the retained last-known-good policy and local adapter. It refuses while a deployment is active: + +```bash +sudo ./scripts/install-deployer.sh \ + --rollback --config /etc/ci-fleet-deployer/deployer.conf +``` + +The report's next action is `restore-host-policy-evidence-then-check`. Restore the prior reviewed `deployer.conf`, exact-head approval evidence, and checkpoint evidence through the same protected operator process that created them; do not copy secrets through the shell. Then verify read-only: + +```bash +sudo ./scripts/install-deployer.sh \ + --check --config /etc/ci-fleet-deployer/deployer.conf +``` + +A cross-environment or cross-target in-place upgrade is rejected. Build a separately isolated host with its own identity and credential scope instead. + +## Submit an approved deployment + +Place one root-owned mode-`0600` request at `/var/lib/ci-fleet-deployer/request.conf`. It uses the same exact-head approval fields shown above. No secret values belong in the request. Before invoking the adapter, runtime revalidates host-role isolation and all protected approval, capability, checkpoint, and production evidence, then permanently consumes the semantic request identity; failed attempts require fresh approval. After success it records the deployed policy as the application-compatible rollback point and atomically moves the request to `last-request.conf`. + +Then run on the deployer host; this changes the application target through its adapter: + +```bash +sudo systemctl start ci-fleet-deployer.service +sudo systemctl status --no-pager ci-fleet-deployer.service +``` + +The runtime and read-only checks serialize on the same flock. Deploy writes a mode-`0600` active-operation marker and removes it on completion. While the adapter runs, `systemd-inhibit` blocks shutdown and sleep; the deploy service has an explicit 45-minute start/stop bound. Upgrade, rollback, drain, and uninstall refuse while that marker belongs to a live or bounded recent process. A root-owned stale marker older than the fixed one-hour recovery bound is removed only by a serialized mutating installer run. Unsafe or malformed stale state fails closed. + +Health runs every five minutes and cleanup daily. Cleanup refuses while drained and is delegated to the application adapter because only application-owned code knows its exact resources. Core itself issues no Docker delete or prune command. + +## Verification and reports + +Verify unit definitions and state without reading protected content: + +```bash +sudo systemd-analyze verify /etc/systemd/system/ci-fleet-deployer*.service \ + /etc/systemd/system/ci-fleet-deployer*.timer +sudo systemctl is-enabled ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer +sudo systemctl is-active ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer +sudo stat -c '%U:%G %a %n' \ + /etc/ci-fleet-deployer \ + /var/lib/ci-fleet-deployer \ + /var/lock/ci-fleet-deployer \ + /var/log/ci-fleet-deployer +``` + +Every installer exit emits one stable secret-free report. Example: + +```text +REPORT action=install result=CHANGED environment=staging target=example-staging version=2222222222222222222222222222222222222222 digest=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa health=healthy changed=yes rollback_available=no next=run-check +``` + +`result` is one of `CHANGED`, `NO_CHANGE`, `BLOCKED`, or `FAILED`. `BLOCKED` means an operator prerequisite or safety gate must be resolved. `FAILED` means attempted candidate work failed. The report includes identifiers, never credential contents. + +## Failure recovery + +1. Do not delete the lock, state, or current release blindly. +2. Confirm whether `/var/lib/ci-fleet-deployer/active-operation` names a live process. +3. If live, wait or follow the application adapter's documented bounded recovery. Never interrupt an unknown deployment to force an upgrade. +4. If the installer reports safe stale state, rerun the same mutating command; it removes only a protected stale marker and current-core staging directory while holding the lock. +5. If candidate validation or activation failed, run `--check`; the previous active release remains selected. +6. If application health failed after an authorized deployment, run `--rollback`, then `--check`. +7. If machine recovery is required, use the checkpoint named in the protected evidence or rebuild the isolated host from reviewed inputs. Core does not operate the hypervisor. + +## Uninstall + +Uninstall changes the host, drains first by refusing any active deployment, disables timers, removes only ci-fleet deployer units and the current activation pointer, and is idempotent: + +```bash +sudo ./scripts/install-deployer.sh \ + --uninstall --config /etc/ci-fleet-deployer/deployer.conf +``` + +It retains configuration, credential references and files, evidence, immutable releases, audit log, and last-known-good state. Review retention policy and destroy those items separately only after access revocation and audit requirements are satisfied. It never unregisters a runner, modifies GitHub, deletes an application repository, or removes unrelated Docker resources. + +## Repository-only validation + +The deterministic test uses an explicit alternate-root test mode and mocked Docker/systemd/network commands. It never changes the test machine's Docker daemon, systemd, GitHub state, or a deployment target: + +```bash +scripts/test-install-deployer.sh +``` + +Prepared-host proof is still required before production use: exercise the reviewed adapter, Docker/systemd behavior, network policy, checkpoint recovery, immutable artifact verification, staging and production isolation, active-deployment drain, health, rollback, and scoped cleanup on a separately authorized isolated deployment host. diff --git a/docs/README.md b/docs/README.md index 72c197c..2ab2cc5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ New operator? Follow the [Quickstart](QUICKSTART.md): what ci-fleet does, instal | Make a project compliant | [Project CI standard](PROJECT-STANDARD.md) and [compliance checklist](COMPLIANCE-CHECKLIST.md) | | Split tests across parallel workers | [Project CI standard](PROJECT-STANDARD.md) and the [parallel workflow example](../examples/workflows/parallel-ci.yml.example) | | Configure automatic updates and cleanup | [Host maintenance](HOST-MAINTENANCE.md) | +| Install or operate a dedicated deployment host | [Deployer host installation](DEPLOYER-HOST.md) | | Monitor hosts and detect missed reports | [Fleet health monitoring](HEALTH-MONITORING.md) and [authenticated status reporting](STATUS-REPORTING.md) | | Handle GitHub App, workflow, or deployment secrets | [Secrets model](SECRETS.md) and [security policy](../SECURITY.md) | | Review accepted implementation scope | [Design decisions](DESIGN-DECISIONS.md) | @@ -42,6 +43,7 @@ New operator? Follow the [Quickstart](QUICKSTART.md): what ci-fleet does, instal | Test shard | One bounded slice of a larger test suite, designed to run independently and usually finish within five minutes. | | Private delivery configuration | Repository allowlists, logical controllers, capacity budgets, environment policy, and required secret names kept in a private but secret-free repository. | | Desired state | A reviewed schema-v3 declaration of runner pools and logical controllers stored in a private, secret-free Git repository. | +| Deployer host | A dedicated non-runner Linux host that applies exact-head approved immutable artifacts through application-owned deployment logic. | ## Design and boundaries @@ -72,6 +74,7 @@ These pages are normative for compatible projects and hosts: - [Run the live pilot](LIVE-PILOT.md) - [Add a host](ADDING-A-HOST.md) - [Install, adopt, check, upgrade, roll back, or remove a controller](DESIRED-STATE.md) +- [Install, check, repair, upgrade, drain, roll back, or remove a deployer](DEPLOYER-HOST.md) - [Add a project](ADDING-A-PROJECT.md) - [Deploy the current experimental prototype](DEPLOYMENT-PROTOTYPE.md) - [Maintain, drain, clean, update, and reboot hosts](HOST-MAINTENANCE.md) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh new file mode 100755 index 0000000..ba4c1e0 --- /dev/null +++ b/scripts/deployer-runtime.sh @@ -0,0 +1,598 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set +x + +operation=${1:-} +case "$operation" in health|cleanup|deploy|drain) ;; *) printf 'ERROR: usage: deployer-runtime.sh health|cleanup|deploy|drain\n' >&2; exit 2 ;; esac + +root=${CI_FLEET_DEPLOYER_ROOT:-} +testing=${CI_FLEET_DEPLOYER_TESTING:-0} +[[ -z "$root" || "$testing" == 1 ]] || { printf 'ERROR: alternate root is test-only\n' >&2; exit 2; } +if [[ "$testing" == 1 && -z "$root" ]]; then printf 'ERROR: test mode requires an alternate root\n' >&2; exit 2; fi +root_path() { printf '%s%s' "$root" "$1"; } +config=${CI_FLEET_DEPLOYER_CONFIG:-$(root_path /var/lib/ci-fleet-deployer/active-policy.conf)} +request=${CI_FLEET_DEPLOYER_REQUEST:-$(root_path /var/lib/ci-fleet-deployer/request.conf)} +state_root=$(root_path /var/lib/ci-fleet-deployer) +log_root=$(root_path /var/log/ci-fleet-deployer) +lock_dir=$(root_path /var/lock/ci-fleet-deployer) +evidence_dir=$(root_path /etc/ci-fleet-deployer/evidence) +deployer_etc=$(root_path /etc/ci-fleet-deployer) +credential_dir=$deployer_etc/credentials +active=$state_root/active-operation +drained=$state_root/drained +last_request=$state_root/last-request.conf +consumed_root=$state_root/consumed-requests +install_state=$state_root/install-state.json +deployed_root=$state_root/deployed +deployed_current=$deployed_root/current +audit_log=$log_root/audit.log +systemd_root=$(root_path /etc/systemd/system) + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 2; } +restore_encoded_file() { + local destination=$1 encoded=$2 temporary + temporary=$(mktemp "$state_root/.restore.XXXXXX") || return + printf '%s' "$encoded" | base64 -d >"$temporary" || { rm -f "$temporary"; return 1; } + chown "$expected_uid" "$temporary" || { rm -f "$temporary"; return 1; } + chmod 0600 "$temporary" || { rm -f "$temporary"; return 1; } + mv -Tf "$temporary" "$destination" +} +if [[ $operation == deploy && -z ${CI_FLEET_DEPLOYER_INHIBITED:-} ]]; then + export CI_FLEET_DEPLOYER_INHIBITED=1 + [[ $testing != 1 || -z ${CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG:-} ]] || printf '%s\n' deploy >>"$CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG" + exec systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ + --why='approved deployment is active' -- "$0" deploy +fi +deploy_exit() { + local status=$? + local recorded_status=${adapter_status:-$status} + local target state_root_safe=1 + # The adapter may recursively clear its writable state root. Recreate only + # an absent boundary; an unsafe replacement remains a hard failure. + if [[ ${audit_pending:-0} == 1 && ! -e "$state_root" && ! -L "$state_root" ]]; then + install -d -m 0700 "$state_root" 2>/dev/null || { status=1; state_root_safe=0; } + fi + if [[ -e "$state_root" || -L "$state_root" ]]; then + [[ -d "$state_root" && ! -L "$state_root" && $(stat -c '%u:%a' "$state_root" 2>/dev/null) == "$expected_uid:700" ]] || { status=1; state_root_safe=0; } + fi + # An adapter that deletes its own consumption marker must not defeat replay + # protection; restore the durable marker rather than clearing audit_pending. + if [[ ${audit_pending:-0} == 1 && -n ${consumed_marker:-} && ! -e "$consumed_marker" ]]; then + [[ ! -e "$consumed_root" && ! -L "$consumed_root" ]] || secure_directory "$consumed_root" 'consumed request directory' + [[ -e "$consumed_root" ]] || install -d -m 0700 "$consumed_root" 2>/dev/null || true + install -m 0600 /dev/null "$consumed_marker" 2>/dev/null || true + sync -f "$consumed_marker" 2>/dev/null || sync "$consumed_marker" 2>/dev/null || status=1 + sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || status=1 + fi + if [[ ${audit_pending:-0} == 1 ]]; then + # If the adapter replaced the audit log, restore the durable prefix copy + # (or the opened inode) under its name before appending the terminal + # failure record, so the record is never written only to an unlinked inode. + if [[ -e "$audit_log" && ! -L "$audit_log" && $(stat -c '%u:%a' "$audit_log" 2>/dev/null) != "$expected_uid:600" ]]; then + chown "$expected_uid" "$audit_log" 2>/dev/null || status=1 + chmod 0600 "$audit_log" 2>/dev/null || status=1 + fi + if [[ ! -e "$audit_log" || $(stat -Lc '%d:%i' /proc/self/fd/8 2>/dev/null) != $(stat -c '%d:%i' "$audit_log" 2>/dev/null) ]]; then + rm -f -- "$audit_log" + if [[ -n ${audit_prefix_copy:-} && -f $audit_prefix_copy ]]; then + cat "$audit_prefix_copy" >"$audit_log" 2>/dev/null || status=1 + else + cat /proc/self/fd/8 >"$audit_log" 2>/dev/null || status=1 + fi + chmod 0600 "$audit_log" 2>/dev/null || true + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=failed phase=%s status=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ + "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ + "${audit_phase:-post-consumption}" "$recorded_status" >>"$audit_log" || status=1 + else + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=failed phase=%s status=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ + "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ + "${audit_phase:-post-consumption}" "$recorded_status" >&8 || status=1 + fi + sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || status=1 + [[ -z ${audit_prefix_copy:-} ]] || rm -f "$audit_prefix_copy" + fi + # On failure, restore the validated incumbent deployed pointer before the + # active-operation guard is durably cleared — but only when the new pointer + # was never published. After publication the new pointer is the truth: the + # application has changed and restoring the incumbent would falsify state. + if [[ ${audit_pending:-0} == 1 && -n ${incumbent_pointer:-} && ${snapshot_pointer:-} != "$deployed_current" ]]; then + # Restore the incumbent snapshot content when the adapter deleted it. + if [[ -n ${incumbent_backup:-} ]]; then + if [[ -e "$deployed_root/$incumbent_pointer" || -L "$deployed_root/$incumbent_pointer" ]]; then + [[ -d "$deployed_root/$incumbent_pointer" && ! -L "$deployed_root/$incumbent_pointer" ]] || { rm -rf -- "${deployed_root:?}/$incumbent_pointer"; mkdir -m 0700 "$deployed_root/$incumbent_pointer"; } + chown "$expected_uid" "$deployed_root/$incumbent_pointer" 2>/dev/null || status=1 + chmod 0700 "$deployed_root/$incumbent_pointer" 2>/dev/null || status=1 + fi + for incumbent_file in policy.conf state.json; do + if [[ ! -f "$deployed_root/$incumbent_pointer/$incumbent_file" || -L "$deployed_root/$incumbent_pointer/$incumbent_file" ]] || ! cmp -s "$incumbent_backup/$incumbent_file" "$deployed_root/$incumbent_pointer/$incumbent_file"; then + mkdir -m 0700 "$deployed_root/$incumbent_pointer" 2>/dev/null || true + install -m 0600 "$incumbent_backup/$incumbent_file" "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 + fi + chown "$expected_uid" "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 + chmod 0600 "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 + done + fi + if [[ ! -L "$deployed_current" || $(readlink "$deployed_current") != "$incumbent_pointer" ]]; then + rm -f -- "$deployed_current" + ln -s "$incumbent_pointer" "$deployed_current" 2>/dev/null || status=1 + sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || status=1 + fi + fi + [[ -z ${incumbent_backup:-} ]] || rm -rf -- "$incumbent_backup" + if [[ ${lkg_backed_up:-0} == 1 && $state_root_safe == 1 ]]; then + restore_encoded_file "$state_root/last-known-good.json" "$lkg_state_backup" 2>/dev/null || status=1 + restore_encoded_file "$state_root/last-known-good-policy.conf" "$lkg_policy_backup" 2>/dev/null || status=1 + fi + if [[ ${audit_pending:-0} == 1 && ${last_request_backed_up:-0} == 1 && $state_root_safe == 1 ]]; then + restore_encoded_file "$last_request" "$last_request_backup" 2>/dev/null || status=1 + fi + rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true + sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || status=1 + if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then + target=$(readlink "$deployed_current" 2>/dev/null || true) + if [[ $target != "${snapshot##*/}" ]]; then rm -rf -- "$snapshot"; fi + fi + if [[ -n ${snapshot_pointer:-} && $snapshot_pointer != "$deployed_current" ]]; then rm -f "$snapshot_pointer"; fi + return "$status" +} +expected_uid=0 +[[ "$testing" != 1 ]] || expected_uid=$(id -u) +# Docker checks must target the host's local daemon, not an inherited remote +# DOCKER_HOST or selected context. +DOCKER_HOST=unix://$(root_path /run/docker.sock) +export DOCKER_HOST +unset DOCKER_CONTEXT +declare -A production=() +secure_file() { + local path=$1 description=$2 mode=${3:-600} + [[ ! -L "$path" && -f "$path" ]] || die "$description must be a regular file, not a symlink" + [[ "$path" == "$(realpath -m -- "$path")" ]] || die "$description path contains a symlink or non-canonical component" + [[ $(realpath -e -- "$path") == $(realpath -m -- "$path") ]] || die "$description path contains a symlink" + [[ $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || die "$description has unsafe owner or mode" +} +secure_directory() { + local path=$1 description=$2 + [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:700" ]] || die "$description has unsafe owner, mode, or type" +} +reject_mixed_role() { + local unit runner_unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" + for unit in ci-fleet-health.service ci-fleet-health.timer ci-fleet-reconcile.service ci-fleet-reconcile.timer ci-fleet-cleanup.service ci-fleet-cleanup.timer ci-fleet-drift.service ci-fleet-drift.timer actions.runner.service; do + [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" && ! -L "$systemd_root/timers.target.wants/$unit" ]] || die 'ordinary CI controller or runner state is present' + done + shopt -s nullglob + for runner_unit in "$systemd_root"/actions.runner.*.service "$systemd_root"/multi-user.target.wants/actions.runner.*.service; do + shopt -u nullglob + [[ -n "$runner_unit" ]] && die 'ordinary GitHub Actions runner service is present' + done + shopt -u nullglob + for path in "$(root_path /etc/ci-fleet/ci-fleet.env)" "$(root_path /etc/ci-fleet/host.env)" "$(root_path /etc/ci-fleet/secrets)" "$(root_path /opt/ci-fleet/current)" "$(root_path /var/lib/ci-fleet/install-state.json)"; do + [[ ! -e "$path" && ! -L "$path" ]] || die 'ordinary CI controller or runner state is present' + done + output=$(docker ps -a --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || die 'Docker workload inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || die 'unrelated Docker workload is present'; done <<<"$output" + output=$(docker network ls --filter type=custom --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || die 'Docker network inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || die 'incompatible custom Docker network is present'; done <<<"$output" + output=$(docker volume ls --format '{{.Name}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || die 'Docker volume inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || die 'incompatible Docker volume is present'; done <<<"$output" +} +inside() { + local path=$1 base=$2 + [[ $(realpath -m -- "$path") == "$(realpath -m -- "$base")/"* ]] +} +valid_utc() { + local value=$1 + [[ "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] && [[ $(date -u -d "$value" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null) == "$value" ]] +} +not_drained() { + if [[ -e "$drained" || -L "$drained" ]]; then + secure_file "$drained" 'drain marker' + die 'deployer is drained' + fi +} +parse_file() { + local path=$1 prefix=$2 kind=$3 allowed=$4 line key value + declare -gA "$prefix=()" + local -n output=$prefix + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -n "$line" && ${line:0:1} != '#' ]] || continue + [[ "$line" == *=* ]] || die "malformed $kind line" + key=${line%%=*}; value=${line#*=} + [[ "$key" =~ ^[A-Z][A-Z0-9_]*$ && -n "$value" && "$value" != *$'\r'* ]] || die "malformed $kind line" + [[ " $allowed " == *" $key "* ]] || die "unknown $kind key" + [[ ! -v "output[$key]" ]] || die "duplicate $kind key" + # key indexes a nameref to an associative array. + # shellcheck disable=SC2004 + output[$key]=$value + done <"$path" +} + +validate_credential() { + secure_directory "$deployer_etc" 'deployer configuration directory' + secure_directory "$credential_dir" 'credential directory' + [[ ${cfg[CREDENTIAL_SCOPE]} == "${cfg[ENVIRONMENT]}" ]] || die 'credential scope does not match the deployment environment' + case ${cfg[CREDENTIAL_PROVIDER]} in + file) + inside "${cfg[CREDENTIAL_REF]}" "$credential_dir" || die 'credential reference is outside the protected credential directory' + secure_file "${cfg[CREDENTIAL_REF]}" 'credential file' + ;; + external) [[ ${cfg[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die 'external credential reference is malformed' ;; + *) die 'unsupported credential provider' ;; + esac +} + +secure_directory "$state_root" 'deployer state directory' +secure_directory "$lock_dir" 'deployer lock directory' +if [[ -z ${CI_FLEET_DEPLOYER_REEXEC:-} ]]; then + exec 9<"$lock_dir" + flock -n 9 || die 'another deployer operation is running' +else + # The re-executed process inherits the locked descriptor 9 from its parent; + # reopening it would drop the lock during the handoff. + : +fi +shopt -s nullglob +transactions=("$state_root"/.transaction.*) +shopt -u nullglob +((${#transactions[@]} == 0)) || die 'interrupted installer transaction requires recovery' + +if [[ "$operation" == drain ]]; then + [[ ! -e "$active" && ! -L "$active" ]] || die 'active deployment prevents drain' + temporary=$(mktemp "$state_root/.drained.XXXXXX") + chmod 0600 "$temporary" + mv -Tf "$temporary" "$drained" + sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || die 'drain marker is not durable' + sync -f "$state_root" 2>/dev/null || die 'drain marker publication is not durable' + exit 0 +fi + +secure_file "$config" 'deployer configuration' +policy_snapshot=$(mktemp "$state_root/.active-policy.XXXXXX") +install -m 0600 "$config" "$policy_snapshot" +config=$policy_snapshot +trap 'rm -f "${policy_snapshot:-}"' EXIT +config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' +parse_file "$config" cfg configuration "$config_keys" +for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done +[[ ${cfg[SCHEMA_VERSION]:-} == 1 ]] || die 'configuration has an unsupported or missing schema version' +[[ ${cfg[CORE_REF]:-} =~ ^[0-9a-f]{40}$ ]] || die 'configuration is missing a valid core revision' +# A concurrent upgrade may switch current between ExecStart resolution and the +# lock; re-exec the release matching the active policy before trusting it. +own_realpath=$(realpath -e -- "${BASH_SOURCE[0]}") +if [[ $own_realpath == */releases/* ]]; then + install_prefix=${own_realpath%%/releases/*} + own_release=${own_realpath#*/releases/} + own_release=${own_release%%/*} + if [[ $own_release != "${cfg[CORE_REF]}" ]]; then + selected=$install_prefix/releases/${cfg[CORE_REF]}/scripts/deployer-runtime.sh + [[ -x $selected && ! -L $selected ]] || die 'active policy revision runtime is unavailable' + [[ -z ${CI_FLEET_DEPLOYER_REEXEC:-} ]] || die 'runtime re-exec did not select the active revision' + # exec replaces this process without running the EXIT trap; the policy + # snapshot is passed by content to the new process via the environment, so + # remove the unmanaged copy before re-exec. + rm -f "$policy_snapshot" + policy_snapshot= + export CI_FLEET_DEPLOYER_REEXEC=1 + exec "$selected" "$operation" + fi +fi +[[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' +[[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' +inside "${cfg[ADAPTER_PATH]}" "$(root_path /etc/ci-fleet-deployer/adapters)" || die 'application adapter is outside the protected adapter directory' +secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 +exec 7<"${cfg[ADAPTER_PATH]}" +[[ $(sha256sum /proc/$$/fd/7 | cut -d' ' -f1) == "${cfg[ADAPTER_SHA256]}" ]] || die 'application adapter digest mismatch' +adapter_path=/proc/$$/fd/7 +validate_credential + +secure_directory "$log_root" 'deployer log directory' + +case "$operation" in + health) + reject_mixed_role + validate_credential + env CI_FLEET_DEPLOYER_CONFIG="$config" "$adapter_path" "$operation" + ;; + cleanup) + not_drained + reject_mixed_role + validate_credential + # Scheduled cleanup mutates application-owned resources; production paths + # remain separately gated like deploy. + [[ ${cfg[ENVIRONMENT]} != production ]] || die 'production cleanup is not authorized by the current accepted scope' + env CI_FLEET_DEPLOYER_CONFIG="$config" "$adapter_path" cleanup + ;; + deploy) + not_drained + [[ ! -e "$active" && ! -L "$active" ]] || die 'active operation marker requires recovery' + reject_mixed_role + secure_file "$request" 'deployment request' + request_snapshot=$(mktemp "$state_root/.request.XXXXXX") + install -m 0600 "$request" "$request_snapshot" + request_snapshot_sha=$(sha256sum "$request_snapshot" | cut -d' ' -f1) + trap 'rm -f "${request_snapshot:-}" "${policy_snapshot:-}"' EXIT + trap 'exit 2' INT TERM + request_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' + parse_file "$request_snapshot" req request "$request_keys" + for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT; do + [[ -v "req[$key]" ]] || die "deployment request is missing $key" + done + [[ ${req[SCHEMA_VERSION]} == 1 ]] || die 'unsupported deployment request schema' + for key in ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do + [[ ${req[$key]} == "${cfg[$key]}" ]] || die "deployment request $key does not match installed policy" + done + [[ ${req[SOURCE_COMMIT]} =~ ^[0-9a-f]{40}$ && ${req[ARTIFACT_IMAGE]} =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?/[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || die 'deployment request is not immutable and qualified' + for key in APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID; do [[ ${req[$key]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ ]] || die "deployment request has an unsafe $key"; done + valid_utc "${req[APPROVED_AT]}" || die 'deployment request has an invalid approval time' + if [[ -e "$last_request" || -L "$last_request" ]]; then + secure_file "$last_request" 'last completed deployment request' + parse_file "$last_request" completed 'last completed deployment request' "$request_keys" + replay=1 + for key in $request_keys; do + [[ -v "completed[$key]" && ${req[$key]} == "${completed[$key]}" ]] || replay=0 + done + ((replay == 0)) || die 'deployment request was already completed' + fi + inside "${cfg[CHECKPOINT_EVIDENCE_PATH]}" "$evidence_dir" || die 'checkpoint evidence is outside the protected evidence directory' + secure_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" 'checkpoint evidence' + checkpoint_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID CHECKPOINT_ID RECORDED_AT' + parse_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" checkpoint 'checkpoint evidence' "$checkpoint_keys" + for key in $checkpoint_keys; do [[ -v "checkpoint[$key]" ]] || die "checkpoint evidence is missing $key"; done + [[ ${checkpoint[SCHEMA_VERSION]} == 1 && ${checkpoint[ENVIRONMENT]} == "${req[ENVIRONMENT]}" && ${checkpoint[TARGET_ID]} == "${req[TARGET_ID]}" ]] || die 'checkpoint evidence does not match the deployment target' + if [[ ! ${checkpoint[CHECKPOINT_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${checkpoint[RECORDED_AT]}"; then die 'checkpoint evidence is malformed'; fi + [[ -v 'cfg[APPROVAL_EVIDENCE_PATH]' ]] || die 'installed policy is missing approval evidence' + inside "${cfg[APPROVAL_EVIDENCE_PATH]}" "$evidence_dir" || die 'approval evidence is outside the protected evidence directory' + secure_file "${cfg[APPROVAL_EVIDENCE_PATH]}" 'approval evidence' + approval_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' + parse_file "${cfg[APPROVAL_EVIDENCE_PATH]}" approved 'approval evidence' "$approval_keys" + for key in $approval_keys; do + [[ -v "approved[$key]" && ${req[$key]} == "${approved[$key]}" ]] || die "deployment request does not match protected approval $key" + done + case ${cfg[APPROVAL_PROVIDER]} in + manual-exact-head|external-exact-head) ;; + github-environment) + [[ -v 'cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]' ]] || die 'GitHub Environment approval is missing capability evidence' + inside "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" "$evidence_dir" || die 'capability evidence is outside the protected evidence directory' + secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' + capability_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' + parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' "$capability_keys" + for key in $capability_keys; do [[ -v "capability[$key]" ]] || die "capability evidence is missing $key"; done + [[ ${capability[ENVIRONMENT]} == "${req[ENVIRONMENT]}" && ${capability[TARGET_ID]} == "${req[TARGET_ID]}" ]] || die 'GitHub Environment capability evidence does not match this installation' + [[ ${capability[SCHEMA_VERSION]} == 1 && ${capability[ENVIRONMENT_PROTECTION]} == verified && ${capability[EXACT_HEAD]} == "${req[SOURCE_COMMIT]}" ]] || die 'GitHub Environment capability evidence is not exact-head verified' + if [[ ! ${capability[CAPABILITY_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${capability[CHECKED_AT]}"; then die 'GitHub Environment capability evidence is malformed'; fi + ;; + *) die 'unsupported approval provider' ;; + esac + if [[ ${cfg[ENVIRONMENT]} == production ]]; then + # Production deployment paths remain separately gated + # (docs/DESIGN-DECISIONS.md); no accepted decision enables them yet. + die 'production deployment is not authorized by the current accepted scope' + fi + if [[ -e "$consumed_root" || -L "$consumed_root" ]]; then secure_directory "$consumed_root" 'consumed request directory'; else install -d -m 0700 "$consumed_root"; fi + request_id=$(for key in $request_keys; do printf '%s=%s\0' "$key" "${req[$key]}"; done | sha256sum | cut -d' ' -f1) + consumed_marker=$consumed_root/$request_id + [[ ! -e "$consumed_marker" && ! -L "$consumed_marker" ]] || die 'deployment request was already consumed' + if [[ -e "$audit_log" || -L "$audit_log" ]]; then secure_file "$audit_log" 'deployer audit log'; else install -m 0600 /dev/null "$audit_log"; fi + exec 8>>"$audit_log" + secure_file "$install_state" 'deployer install state' + python3 - "$install_state" "${cfg[CORE_REF]}" "${cfg[ENVIRONMENT]}" "${cfg[TARGET_ID]}" "${cfg[DEPLOYER_IDENTITY]}" "${cfg[SOURCE_COMMIT]}" "${cfg[ARTIFACT_IMAGE]}" <<'PY' >/dev/null 2>&1 || die 'deployer install state is malformed' +import json, sys +try: value = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): raise SystemExit(1) +keys = ('core_ref', 'environment', 'target', 'deployer_identity', 'source_commit', 'artifact') +raise SystemExit(0 if all(value.get(k) == v for k, v in zip(keys, sys.argv[2:])) else 1) +PY + [[ ${cfg[CORE_REF]} =~ ^[0-9a-f]{40}$ ]] || die 'configuration is missing a valid core revision' + if [[ -e "$deployed_root" || -L "$deployed_root" ]]; then secure_directory "$deployed_root" 'deployed snapshot directory'; else install -d -m 0700 "$deployed_root"; fi + [[ -e "$deployed_current" || -L "$deployed_current" ]] || die 'deployed rollback snapshot is missing' + # The rollback path consumes the retained last-known-good pair, not the + # deployed snapshot; once that pair exists it must be complete and valid + # before approval consumption. + previous_state=$state_root/last-known-good.json + previous_policy=$state_root/last-known-good-policy.conf + if [[ -e "$previous_state" || -L "$previous_state" || -e "$previous_policy" || -L "$previous_policy" ]]; then + secure_file "$previous_state" 'last-known-good state' + secure_file "$previous_policy" 'last-known-good policy' + declare -A lkg_policy=() + parse_file "$previous_policy" lkg_policy 'last-known-good policy' "$config_keys" + [[ ${lkg_policy[CORE_REF]:-} =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good policy has an invalid core revision' + # Validate the entire retained release as perform_rollback's + # release_complete does: ownership, modes, types, and tree digest. + lkg_release=$(root_path /opt/ci-fleet-deployer)/releases/${lkg_policy[CORE_REF]} + [[ -d "$lkg_release" && ! -L "$lkg_release" && $(stat -c '%u:%a' "$lkg_release") == "$expected_uid:755" ]] || die 'last-known-good release is incomplete' + for dir in "$lkg_release/scripts" "$lkg_release/deploy" "$lkg_release/deploy/deployer"; do [[ -d "$dir" && ! -L "$dir" && $(stat -c '%u:%a' "$dir") == "$expected_uid:755" ]] || die 'last-known-good release is incomplete'; done + for entry in "$lkg_release/scripts/install-deployer.sh" "$lkg_release/scripts/deployer-runtime.sh"; do [[ ! -L "$entry" && -f "$entry" && $(stat -c '%u:%a' "$entry") == "$expected_uid:755" ]] || die 'last-known-good release is incomplete'; done + [[ -f "$lkg_release/.ci-fleet-tree-sha256" ]] || die 'last-known-good release is incomplete' + [[ $(<"$lkg_release/.ci-fleet-tree-sha256") == "$(cd "$lkg_release" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1)" ]] || die 'last-known-good release tree digest mismatch' + for entry in "$lkg_release"/deploy/deployer/*; do [[ -f "$entry" && ! -L "$entry" && $(stat -c '%u:%a' "$entry") == "$expected_uid:644" ]] || die 'last-known-good release is incomplete'; done + # Cross-validate the retained pair exactly as the rollback path does: + # state must match policy, and the retained adapter must match its pin. + python3 - "$previous_state" "$previous_policy" <<'PY' >/dev/null 2>&1 || die 'last-known-good state and policy do not cross-validate' +import json, re, sys +try: + state = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): + raise SystemExit(1) +policy = {} +try: + for line in open(sys.argv[2], encoding='utf-8'): + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + key, sep, value = line.partition('=') + if sep and key not in policy: + policy[key] = value +except OSError: + raise SystemExit(1) +sha = re.compile(r'^[0-9a-f]{40}$') +pairs = (('core_ref','CORE_REF'), ('environment','ENVIRONMENT'), ('target','TARGET_ID'), ('source_commit','SOURCE_COMMIT'), ('artifact','ARTIFACT_IMAGE'), ('deployer_identity','DEPLOYER_IDENTITY')) +ok = all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) +ok = ok and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) +ok = ok and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact'])) +ok = ok and policy.get('CREDENTIAL_SCOPE') == policy.get('ENVIRONMENT') +raise SystemExit(0 if ok else 1) +PY + [[ -v 'lkg_policy[ADAPTER_PATH]' && -v 'lkg_policy[ADAPTER_SHA256]' ]] || die 'last-known-good policy is missing its adapter pin' + inside "${lkg_policy[ADAPTER_PATH]}" "$deployer_etc/adapters" || die 'last-known-good adapter is outside the protected adapter directory' + [[ ! -L "${lkg_policy[ADAPTER_PATH]}" && -f "${lkg_policy[ADAPTER_PATH]}" && $(stat -c '%u:%a' "${lkg_policy[ADAPTER_PATH]}") == "$expected_uid:700" ]] || die 'last-known-good adapter is missing or unsafe' + [[ $(sha256sum "${lkg_policy[ADAPTER_PATH]}" | cut -d' ' -f1) == "${lkg_policy[ADAPTER_SHA256]}" ]] || die 'last-known-good adapter digest does not match its retained policy' + fi + [[ -L "$deployed_current" ]] || die 'deployed snapshot pointer is absent or unsafe' + [[ $(readlink "$deployed_current") =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'deployed snapshot pointer target is not canonical' + deployed_snapshot=$(readlink -f "$deployed_current") + inside "$deployed_snapshot" "$deployed_root" || die 'deployed snapshot pointer escapes managed state' + secure_directory "$deployed_snapshot" 'deployed snapshot' + secure_file "$deployed_snapshot/policy.conf" 'deployed rollback policy' + secure_file "$deployed_snapshot/state.json" 'deployed rollback state' + # Deploying consumes the approval; the retained rollback point must be + # fully usable first, or the change proceeds with no way back. + declare -A deployed_policy=() + parse_file "$deployed_snapshot/policy.conf" deployed_policy 'deployed rollback policy' "$config_keys" + for key in ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE ENVIRONMENT TARGET_ID; do [[ -v "deployed_policy[$key]" ]] || die "deployed rollback policy is missing $key"; done + inside "${deployed_policy[ADAPTER_PATH]}" "$deployer_etc/adapters" || die 'deployed rollback adapter is outside the protected adapter directory' + [[ ! -L "${deployed_policy[ADAPTER_PATH]}" && -f "${deployed_policy[ADAPTER_PATH]}" && $(stat -c '%u:%a' "${deployed_policy[ADAPTER_PATH]}") == "$expected_uid:700" ]] || die 'deployed rollback adapter is missing or unsafe' + [[ $(sha256sum "${deployed_policy[ADAPTER_PATH]}" | cut -d' ' -f1) == "${deployed_policy[ADAPTER_SHA256]}" ]] || die 'deployed rollback adapter digest does not match its snapshot policy' + [[ ${deployed_policy[CREDENTIAL_SCOPE]} == "${deployed_policy[ENVIRONMENT]}" ]] || die 'deployed rollback credential scope does not match its environment' + if [[ ${deployed_policy[CREDENTIAL_PROVIDER]} == file ]]; then + inside "${deployed_policy[CREDENTIAL_REF]}" "$credential_dir" || die 'deployed rollback credential is outside the protected credential directory' + secure_file "${deployed_policy[CREDENTIAL_REF]}" 'deployed rollback credential' + elif [[ ${deployed_policy[CREDENTIAL_PROVIDER]} == external ]]; then + [[ ${deployed_policy[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die 'deployed rollback policy has an invalid external secret-manager adapter reference' + else + die 'deployed rollback policy has an unsupported credential provider' + fi + snapshot=$(mktemp -d "$deployed_root/.snapshot.XXXXXX") + chmod 0700 "$snapshot" + install -m 0600 "$config" "$snapshot/policy.conf" + install -m 0600 "$install_state" "$snapshot/state.json" + snapshot_policy_sha=$(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) + snapshot_state_sha=$(sha256sum "$snapshot/state.json" | cut -d' ' -f1) + reject_mixed_role + validate_credential + audit_pending=1 + audit_phase=pre-adapter + adapter_status= + trap deploy_exit EXIT + trap 'exit 2' INT TERM + install -m 0600 /dev/null "$consumed_marker" || die 'deployment request consumption marker failed' + sync -f "$consumed_marker" 2>/dev/null || sync "$consumed_marker" 2>/dev/null || die 'deployment request consumption marker is not durable' + sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || die 'deployment request consumption is not durable' + umask 077 + temporary=$(mktemp "$state_root/.active.XXXXXX") + active_temporary=$temporary + printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" + if [[ $testing != 1 ]]; then + printf 'boot_id=%s\nstart_time=%s\n' "$(>"$temporary" + fi + mv -Tf "$temporary" "$active" + active_temporary= + sync -f "$active" 2>/dev/null || sync "$active" 2>/dev/null || die 'active operation marker is not durable' + sync -f "$state_root" 2>/dev/null || die 'active operation marker publication is not durable' + # Record the durably consumed approval before the adapter runs, so even a + # SIGKILL or power loss mid-adapter leaves the attempt's identities. + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=consumed phase=pre-adapter status=none\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ + "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ + >&8 || die 'deployment consumption audit record failed' + sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || die 'deployment consumption audit record is not durable' + audit_prefix_sha=$(sha256sum "$audit_log" | cut -d' ' -f1) + audit_prefix_copy=$(mktemp "$state_root/.audit-prefix.XXXXXX") + install -m 0600 "$audit_log" "$audit_prefix_copy" + # Preserve the retained rollback pair in unexported process memory. A + # filesystem backup under the adapter-writable state root would be lost to + # the same recursive cleanup as the retained files. + if [[ -f "$previous_state" && -f "$previous_policy" ]]; then + lkg_state_backup=$(base64 -w0 "$previous_state") + lkg_policy_backup=$(base64 -w0 "$previous_policy") + lkg_backed_up=1 + fi + if [[ -f "$last_request" && ! -L "$last_request" ]]; then + last_request_backup=$(base64 -w0 "$last_request") + last_request_backed_up=1 + fi + # Capture the validated incumbent pointer independently of adapter-writable + # state so a failed adapter cannot destroy or falsify the rollback point. + incumbent_pointer=$(readlink "$deployed_current") + incumbent_backup=$(mktemp -d "$state_root/.incumbent.XXXXXX") + install -m 0600 "$deployed_current/policy.conf" "$deployed_current/state.json" "$incumbent_backup/" + set +e + env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy + adapter_status=$? + set -e + # An in-place truncation keeps the same inode; the durable prefix must survive. + if [[ $(sha256sum "$audit_log" | cut -d' ' -f1) != "$audit_prefix_sha" ]]; then + install -m 0600 "$audit_prefix_copy" "$audit_log" 2>/dev/null || true + die 'deployer audit log changed during deployment' + fi + rm -f "$audit_prefix_copy" + if ((adapter_status != 0)); then + audit_phase=adapter + die 'deployment adapter failed after approval consumption' + fi + adapter_status= + audit_phase=post-adapter + # The adapter change is applied and the approval is consumed; a signal from + # here on must not abort before the new snapshot pointer is durable and the + # success audit record is written, or later rollback would use stale state. + trap '' INT TERM + secure_directory "$deployed_root" 'deployed snapshot directory' + inside "$snapshot" "$deployed_root" || die 'prepared deployed snapshot escaped managed state' + secure_directory "$snapshot" 'prepared deployed snapshot' + secure_file "$snapshot/policy.conf" 'prepared deployed policy' + secure_file "$snapshot/state.json" 'prepared deployed state' + [[ $(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) == "$snapshot_policy_sha" && $(sha256sum "$snapshot/state.json" | cut -d' ' -f1) == "$snapshot_state_sha" ]] || die 'prepared deployed snapshot changed during deployment' + # The adapter receives the request snapshot path; only verified bytes may + # become the completed-request record. + secure_file "$request_snapshot" 'deployment request snapshot' + [[ $(sha256sum "$request_snapshot" | cut -d' ' -f1) == "$request_snapshot_sha" ]] || die 'deployment request snapshot changed during deployment' + pointer=$(mktemp -u "$deployed_root/.current.XXXXXX") + snapshot_pointer=$pointer + # Retire the validated pre-adapter incumbent, never an adapter-writable + # pointer target read after the adapter ran. + retired_snapshot=$incumbent_pointer + [[ -z "$retired_snapshot" || "$retired_snapshot" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'current deployed snapshot pointer is unsafe' + ln -s "${snapshot##*/}" "$pointer" + [[ -z ${CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF:-} || $testing != 1 ]] || kill -"$CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF" $$ + sync -f "$snapshot/policy.conf" "$snapshot/state.json" 2>/dev/null || die 'prepared deployed snapshot is not durable' + sync -f "$snapshot" 2>/dev/null || sync "$snapshot" 2>/dev/null || die 'prepared deployed snapshot is not durable' + mv -Tf "$pointer" "$deployed_current" + sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'deployed snapshot pointer is not durable' + snapshot_pointer=$deployed_current + snapshot= + if [[ -f "$request" && ! -L "$request" ]] && cmp -s "$request_snapshot" "$request"; then rm -f "$request"; fi + mv -Tf "$request_snapshot" "$last_request" + request_snapshot= + secure_file "$audit_log" 'deployer audit log' + [[ $(stat -Lc '%d:%i' /proc/self/fd/8) == $(stat -c '%d:%i' "$audit_log") ]] || die 'deployer audit log changed during deployment' + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=success\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ + "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ + >&8 + sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || die 'deployment success audit is not durable' + # An adapter that deleted its consumption marker must not weaken replay + # protection on success either; ensure the durable marker before the exit + # guard is disabled. + if [[ ! -e "$consumed_marker" ]]; then + [[ -e "$consumed_root" ]] || install -d -m 0700 "$consumed_root" + install -m 0600 /dev/null "$consumed_marker" || die 'deployment request consumption marker failed' + sync -f "$consumed_marker" 2>/dev/null || sync "$consumed_marker" 2>/dev/null || die 'deployment request consumption marker is not durable' + sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || die 'deployment request consumption is not durable' + fi + # All fallible commit work is done; restore the retained pair if the + # adapter touched it, then retire the incumbent snapshot. + if [[ ${lkg_backed_up:-0} == 1 ]]; then + restore_encoded_file "$state_root/last-known-good.json" "$lkg_state_backup" || die 'retained rollback state restoration failed' + restore_encoded_file "$state_root/last-known-good-policy.conf" "$lkg_policy_backup" || die 'retained rollback policy restoration failed' + sync -f "$state_root" 2>/dev/null || die 'retained rollback pair restoration is not durable' + lkg_backed_up=0 + fi + if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi + sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'retired deployed snapshot is not durable' + audit_pending=0 + ;; +esac diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh new file mode 100755 index 0000000..5e2be00 --- /dev/null +++ b/scripts/install-deployer.sh @@ -0,0 +1,1360 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set +x +export PYTHONDONTWRITEBYTECODE=1 + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +mode= +config= +validated_config= +error_reported=0 +root=${CI_FLEET_DEPLOYER_ROOT:-} +testing=${CI_FLEET_DEPLOYER_TESTING:-0} +effective_uid=${EUID:-$(id -u)} +action=unknown +environment=unknown +target=unknown +core_ref=unknown +artifact=unknown +health=unknown +staging_path= +checkout_snapshot= +transaction_dir= +transaction_preparing=0 +transaction_committed=0 +recovered_rollback=0 +on_exit() { + local status=$? recovery_status=0 + if [[ -n ${transaction_dir:-} && ${transaction_preparing:-0} == 1 ]]; then + rm -rf -- "$transaction_dir" + transaction_dir= + elif [[ -n ${transaction_dir:-} && ${transaction_committed:-0} != 1 ]]; then + set +e + if [[ -e "$transaction_dir/application-rollback-committed" || -L "$transaction_dir/application-rollback-committed" ]]; then finalize_committed_rollback; recovery_status=$?; else restore_transaction; recovery_status=$?; fi + set -e + if ((recovery_status != 0)); then + status=$recovery_status + printf 'ERROR: transaction recovery failed; retained %s for the next installer recovery\n' "$transaction_dir" >&2 + fi + fi + [[ -z ${staging_path:-} ]] || rm -rf -- "$staging_path" + [[ -z ${checkout_snapshot:-} ]] || rm -rf -- "$checkout_snapshot" + [[ -z ${validated_config:-} ]] || rm -f -- "$validated_config" + [[ -z ${policy_check_snapshot:-} ]] || rm -f -- "$policy_check_snapshot" + if ((status != 0 && error_reported == 0)); then report FAILED no inspect-and-retry "$(rollback_available)" >&2; fi + return "$status" +} +trap on_exit EXIT + +usage() { + cat >&2 <<'EOF' +usage: install-deployer.sh --check|--install|--upgrade|--repair|--rollback|--drain|--resume|--uninstall --config /etc/ci-fleet-deployer/deployer.conf + +Modes are explicit and mutually exclusive. Configuration and credential references +are host-local; secret values are never accepted as arguments. +EOF +} +report() { + local result=$1 changed=$2 next=$3 rollback=${4:-no} + printf 'REPORT action=%s result=%s environment=%s target=%s version=%s digest=%s health=%s changed=%s rollback_available=%s next=%s\n' \ + "$action" "$result" "$environment" "$target" "$core_ref" "${artifact#*@}" "$health" "$changed" "$rollback" "$next" +} +rollback_available() { + [[ -n ${previous_state:-} && -f ${previous_state:-/nonexistent} && ! -L ${previous_state:-/nonexistent} && -n ${previous_policy:-} && -f ${previous_policy:-/nonexistent} && ! -L ${previous_policy:-/nonexistent} ]] || { printf no; return; } + [[ $(stat -c '%u:%a' "$previous_state" 2>/dev/null) == "$expected_uid:600" && $(stat -c '%u:%a' "$previous_policy" 2>/dev/null) == "$expected_uid:600" ]] || { printf no; return; } + python3 - "$previous_state" "$previous_policy" <<'PY' >/dev/null 2>&1 || { printf no; return; } +import json, sys +allowed = {"SCHEMA_VERSION","CORE_REF","ENVIRONMENT","TARGET_ID","DEPLOYER_IDENTITY","ADAPTER_PATH","ADAPTER_SHA256","CREDENTIAL_PROVIDER","CREDENTIAL_REF","CREDENTIAL_SCOPE","APPROVAL_PROVIDER","APPROVAL_EVIDENCE_PATH","APPROVAL_CAPABILITY_EVIDENCE_PATH","PRODUCTION_AUTHORIZATION_EVIDENCE_PATH","CHECKPOINT_EVIDENCE_PATH","SOURCE_COMMIT","ARTIFACT_IMAGE","NETWORK_HOST","MIN_DISK_GIB","REQUIRE_COMPOSE"} +try: + state = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): + raise SystemExit(1) +policy = {} +try: + for line in open(sys.argv[2], encoding='utf-8'): + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + key, sep, value = line.partition('=') + if not sep or key not in allowed or key in policy or not value: + raise SystemExit(1) + policy[key] = value +except OSError: + raise SystemExit(1) +import re +sha = re.compile(r'^[0-9a-f]{40}$') +pairs = (('core_ref','CORE_REF'), ('environment','ENVIRONMENT'), ('target','TARGET_ID'), ('source_commit','SOURCE_COMMIT'), ('artifact','ARTIFACT_IMAGE'), ('deployer_identity','DEPLOYER_IDENTITY')) +ok = (all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) + and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) + and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact'])) + and policy.get('SCHEMA_VERSION') == '1') +raise SystemExit(0 if ok else 1) +PY + local rollback_adapter_path rollback_adapter_sha rollback_credential_provider rollback_credential_ref + rollback_adapter_path=$(awk '$0 ~ /^ADAPTER_PATH=/ {sub(/^ADAPTER_PATH=/, ""); print}' "$previous_policy") + rollback_adapter_sha=$(awk '$0 ~ /^ADAPTER_SHA256=/ {sub(/^ADAPTER_SHA256=/, ""); print}' "$previous_policy") + [[ $rollback_adapter_path == "$etc_root/adapters/"* ]] || { printf no; return; } + [[ ! -L "$rollback_adapter_path" && -f "$rollback_adapter_path" && $(stat -c '%u:%a' "$rollback_adapter_path" 2>/dev/null) == "$expected_uid:700" ]] || { printf no; return; } + [[ $(sha256sum "$rollback_adapter_path" 2>/dev/null | cut -d' ' -f1) == "$rollback_adapter_sha" ]] || { printf no; return; } + rollback_credential_provider=$(awk '$0 ~ /^CREDENTIAL_PROVIDER=/ {sub(/^CREDENTIAL_PROVIDER=/, ""); print}' "$previous_policy") + rollback_credential_ref=$(awk '$0 ~ /^CREDENTIAL_REF=/ {sub(/^CREDENTIAL_REF=/, ""); print}' "$previous_policy") + if [[ $rollback_credential_provider == file ]]; then + [[ $rollback_credential_ref == "$etc_root/credentials/"* ]] || { printf no; return; } + [[ ! -L "$rollback_credential_ref" && -f "$rollback_credential_ref" && $(stat -c '%u:%a' "$rollback_credential_ref" 2>/dev/null) == "$expected_uid:600" ]] || { printf no; return; } + elif [[ $rollback_credential_provider == external ]]; then + [[ $rollback_credential_ref =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || { printf no; return; } + else + printf no; return + fi + local rollback_scope rollback_environment rollback_release + rollback_scope=$(awk '$0 ~ /^CREDENTIAL_SCOPE=/ {sub(/^CREDENTIAL_SCOPE=/, ""); print}' "$previous_policy") + rollback_environment=$(awk '$0 ~ /^ENVIRONMENT=/ {sub(/^ENVIRONMENT=/, ""); print}' "$previous_policy") + [[ $rollback_scope == "$rollback_environment" ]] || { printf no; return; } + rollback_release=$(awk '$0 ~ /^CORE_REF=/ {sub(/^CORE_REF=/, ""); print}' "$previous_policy") + [[ $rollback_release =~ ^[0-9a-f]{40}$ ]] || { printf no; return; } + release_complete "$releases/$rollback_release" 2>/dev/null || { printf no; return; } + # Rollback requires a completed deployment; mirror the perform_rollback gate. + [[ -f "$state_root/last-request.conf" && ! -L "$state_root/last-request.conf" ]] || { printf no; return; } + printf yes +} +die() { error_reported=1; printf 'ERROR: %s\n' "$*" >&2; report FAILED no inspect-and-retry "$(rollback_available)" >&2; exit 2; } +block() { error_reported=1; printf 'BLOCKED: %s\n' "$*" >&2; report BLOCKED no resolve-precondition "$(rollback_available)" >&2; exit 3; } + +while (($#)); do + case "$1" in + --check|--install|--upgrade|--repair|--rollback|--drain|--resume|--uninstall) + [[ -z "$mode" ]] || die 'select exactly one operating mode' + mode=${1#--}; action=$mode; shift ;; + --config) (($# >= 2)) || die '--config requires a value'; config=$2; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done +[[ -n "$mode" ]] || { usage; die 'an explicit operating mode is required'; } +[[ -n "$config" ]] || die '--config is required' +[[ -z "$root" || "$testing" == 1 ]] || die 'CI_FLEET_DEPLOYER_ROOT is test-only' +if [[ "$testing" == 1 ]]; then + [[ -n "$root" ]] || die 'test mode requires an alternate root' + effective_uid=${CI_FLEET_DEPLOYER_EUID_OVERRIDE:-$effective_uid} +else + [[ -z "$root" ]] || die 'alternate root is forbidden' + [[ -z ${CI_FLEET_DEPLOYER_EUID_OVERRIDE:-} ]] || die 'effective UID override is test-only' +fi +if [[ "$mode" != check && "$effective_uid" != 0 ]]; then die 'run this mode as root'; fi + +root_path() { printf '%s%s' "$root" "$1"; } +etc_root=$(root_path /etc/ci-fleet-deployer) +install_root=$(root_path /opt/ci-fleet-deployer) +releases=$install_root/releases +current=$install_root/current +state_root=$(root_path /var/lib/ci-fleet-deployer) +state_file=$state_root/install-state.json +active_policy=$state_root/active-policy.conf +previous_state=$state_root/last-known-good.json +previous_policy=$state_root/last-known-good-policy.conf +deployed_root=$state_root/deployed +deployed_current=$deployed_root/current +drained=$state_root/drained +active_operation=$state_root/active-operation +lock_root=$(root_path /var/lock/ci-fleet-deployer) +log_root=$(root_path /var/log/ci-fleet-deployer) +systemd_root=$(root_path /etc/systemd/system) +unit_source=$repo_root/deploy/deployer +unit_names=( + ci-fleet-deployer.service + ci-fleet-deployer-health.service ci-fleet-deployer-health.timer + ci-fleet-deployer-cleanup.service ci-fleet-deployer-cleanup.timer + ci-fleet-deployer-drain.service +) +timer_names=(ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer) +config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' + +expected_uid=0 +[[ "$testing" != 1 ]] || expected_uid=$(id -u) +inside() { + local path=$1 base=$2 normalized normalized_base + normalized=$(realpath -m -- "$path") + normalized_base=$(realpath -m -- "$base") + [[ "$normalized" == "$normalized_base/"* ]] +} +valid_utc() { + local value=$1 + [[ "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] && [[ $(date -u -d "$value" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null) == "$value" ]] +} +credential_reference_safe() { + local provider=$1 reference=$2 description=$3 + case $provider in + file) + inside "$reference" "$etc_root/credentials" || die "$description credential reference is outside the approved credential directory" + [[ ! -L $reference && -f $reference ]] || die "$description credential reference must be a regular file, not a symlink" + [[ $reference == "$(realpath -m -- "$reference")" && $(realpath -e -- "$reference") == "$reference" ]] || die "$description credential reference contains a symlink or non-canonical component" + [[ $(stat -c '%u:%a' "$reference") == "$expected_uid:600" ]] || die "$description credential file must be owner-only mode 0600" + ;; + external) + [[ $reference =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die "$description has an invalid external secret-manager adapter reference" + ;; + *) die "$description CREDENTIAL_PROVIDER must be file or external" ;; + esac +} + +secure_file() { + local path=$1 description=$2 mode=${3:-600} + [[ ! -L "$path" && -f "$path" ]] || block "$description must be a regular file, not a symlink" + [[ "$path" == "$(realpath -m -- "$path")" ]] || block "$description path contains a symlink or non-canonical component" + [[ $(realpath -e -- "$path") == $(realpath -m -- "$path") ]] || block "$description path contains a symlink" + [[ $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || block "$description must be owned by root with mode 0$mode" +} +secure_directory() { + local path=$1 mode=$2 create=${3:-0} + [[ ! -L "$path" ]] || die "unsafe symlinked managed directory: $path" + if [[ ! -e "$path" ]]; then + [[ "$create" == 1 ]] || return 1 + install -d -m "$mode" "$path" + fi + if [[ "$create" == 1 && -d "$path" && $(stat -c %u "$path") == "$expected_uid" ]]; then chmod "$mode" "$path"; fi + [[ -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || die "unsafe managed directory: $path" +} + +parse_file() { + local path=$1 prefix=$2 kind=$3 allowed=$4 line key value + declare -gA "$prefix=()" + local -n output=$prefix + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -n "$line" && ${line:0:1} != '#' ]] || continue + [[ "$line" == *=* ]] || block "malformed $kind line" + key=${line%%=*}; value=${line#*=} + [[ "$key" =~ ^[A-Z][A-Z0-9_]*$ && -n "$value" && "$value" != *$'\r'* && "$value" != *$'\n'* ]] || block "malformed $kind line" + [[ ! -v "output[$key]" ]] || block "duplicate $kind key: $key" + [[ " $allowed " == *" $key "* ]] || block "unknown $kind key: $key" + # key indexes a nameref to an associative array. + # shellcheck disable=SC2004 + output[$key]=$value + done <"$path" +} + +validate_config() { + inside "$config" "$etc_root" || block "configuration path must be inside $etc_root" + if [[ "$mode" == rollback || "$mode" == uninstall ]]; then + # Rollback and uninstall must work even when the operator-owned candidate + # configuration or its directory is missing or malformed. + declare -gA cfg=() + if [[ ! -e "$etc_root" ]]; then return; fi + secure_directory "$etc_root" 700 0 || block 'configuration directory has an unsafe owner, mode, or type' + if [[ -f "$config" && ! -L "$config" && $(stat -c '%u:%a' "$config" 2>/dev/null) == "$expected_uid:600" ]]; then + validated_config=$(mktemp) + install -m 0600 "$config" "$validated_config" + config=$validated_config + cfg_dump=$(parse_file "$config" cfg configuration "$config_keys" && declare -p cfg) 2>/dev/null || cfg_dump= + if [[ -n "$cfg_dump" ]]; then eval "$cfg_dump"; else cfg=(); fi + if [[ ${cfg[SCHEMA_VERSION]:-} == 1 && ${cfg[ENVIRONMENT]:-} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]:-} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]]; then + environment=${cfg[ENVIRONMENT]}; target=${cfg[TARGET_ID]} + fi + fi + return + fi + secure_directory "$etc_root" 700 0 || block 'configuration directory is missing' + secure_file "$config" 'configuration file' + validated_config=$(mktemp) + install -m 0600 "$config" "$validated_config" + config=$validated_config + parse_file "$config" cfg configuration "$config_keys" + local key candidate_environment candidate_target candidate_core candidate_artifact + for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID; do + [[ -v "cfg[$key]" ]] || block "configuration is missing required key: $key" + done + [[ ${cfg[SCHEMA_VERSION]} == 1 ]] || block 'unsupported configuration schema' + candidate_environment=${cfg[ENVIRONMENT]}; candidate_target=${cfg[TARGET_ID]} + [[ "$candidate_environment" =~ ^[a-z][a-z0-9-]{0,31}$ ]] || block 'invalid explicit environment' + [[ "$candidate_target" =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'invalid explicit target identity' + environment=$candidate_environment; target=$candidate_target + if [[ "$mode" == drain || "$mode" == uninstall || "$mode" == rollback ]]; then return; fi + secure_directory "$etc_root/adapters" 700 0 || block 'adapter directory is missing' + secure_directory "$etc_root/credentials" 700 0 || block 'credential directory is missing' + secure_directory "$etc_root/evidence" 700 0 || block 'evidence directory is missing' + for key in SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE; do + [[ -v "cfg[$key]" ]] || block "configuration is missing required key: $key" + done + [[ ${cfg[DEPLOYER_IDENTITY]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'invalid deployer identity' + [[ ${cfg[CREDENTIAL_SCOPE]} == "$environment" ]] || block 'credential scope must exactly match the explicit environment' + candidate_core=${cfg[CORE_REF]}; candidate_artifact=${cfg[ARTIFACT_IMAGE]} + [[ "$candidate_core" =~ ^[0-9a-f]{40}$ && ${cfg[SOURCE_COMMIT]} =~ ^[0-9a-f]{40}$ ]] || block 'core and source revisions must be full lowercase commit SHAs' + [[ "$candidate_artifact" =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?/[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || block 'artifact image must be an immutable qualified digest reference' + core_ref=$candidate_core; artifact=$candidate_artifact + [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || block 'adapter digest must be lowercase SHA-256' + [[ ${cfg[NETWORK_HOST]} =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$ ]] || block 'invalid network prerequisite host' + [[ ${cfg[MIN_DISK_GIB]} =~ ^[1-9][0-9]{0,3}$ ]] || block 'MIN_DISK_GIB must be a positive integer' + [[ ${cfg[REQUIRE_COMPOSE]} == 0 || ${cfg[REQUIRE_COMPOSE]} == 1 ]] || block 'REQUIRE_COMPOSE must be 0 or 1' + inside "${cfg[ADAPTER_PATH]}" "$etc_root/adapters" || block 'adapter path is outside the approved adapter directory' + secure_file "${cfg[ADAPTER_PATH]}" 'adapter file' 700 + [[ $(sha256sum "${cfg[ADAPTER_PATH]}" | cut -d' ' -f1) == "${cfg[ADAPTER_SHA256]}" ]] || block 'adapter digest does not match the protected regular file' + case ${cfg[CREDENTIAL_PROVIDER]} in + file) + inside "${cfg[CREDENTIAL_REF]}" "$etc_root/credentials" || block 'credential reference is outside the approved credential directory' + [[ ! -L ${cfg[CREDENTIAL_REF]} && -f ${cfg[CREDENTIAL_REF]} ]] || block 'credential reference must be a regular file, not a symlink' + [[ ${cfg[CREDENTIAL_REF]} == "$(realpath -m -- "${cfg[CREDENTIAL_REF]}")" && $(realpath -e -- "${cfg[CREDENTIAL_REF]}") == "${cfg[CREDENTIAL_REF]}" ]] || block 'credential reference contains a symlink or non-canonical component' + [[ $(stat -c '%u:%a' "${cfg[CREDENTIAL_REF]}") == "$expected_uid:600" ]] || block 'credential file must be owner-only mode 0600' + ;; + external) + [[ ${cfg[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || block 'invalid external secret-manager adapter reference' + ;; + *) block 'CREDENTIAL_PROVIDER must be file or external' ;; + esac + validate_evidence + validate_production_gate +} + +validate_production_gate() { + local key + if [[ "$environment" != production ]]; then + [[ ! -v 'cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]' ]] || block 'production authorization evidence is forbidden outside production' + return + fi + [[ -v 'cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]' ]] || block 'production requires separate authorization evidence' + inside "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" "$etc_root/evidence" || block 'production authorization evidence is outside the approved evidence directory' + secure_file "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" 'production authorization evidence' + parse_file "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" production_gate 'production authorization evidence' 'SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE AUTHORIZED_BY GATE_ID AUTHORIZED_AT' + for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE AUTHORIZED_BY GATE_ID AUTHORIZED_AT; do + [[ -v "production_gate[$key]" ]] || block "production authorization evidence is missing $key" + done + [[ ${production_gate[SCHEMA_VERSION]} == 1 && ${production_gate[ENVIRONMENT]} == production ]] || block 'production authorization evidence has the wrong scope' + for key in TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do [[ ${production_gate[$key]} == "${cfg[$key]}" ]] || block "production authorization evidence does not match exact $key"; done + [[ ${production_gate[AUTHORIZED_BY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${production_gate[GATE_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || block 'production authorization identity is malformed' + valid_utc "${production_gate[AUTHORIZED_AT]}" || block 'production authorization timestamp must be UTC RFC3339' +} + +validate_evidence() { + local allowed='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' + inside "${cfg[APPROVAL_EVIDENCE_PATH]}" "$etc_root/evidence" || block 'approval evidence is outside the approved evidence directory' + secure_file "${cfg[APPROVAL_EVIDENCE_PATH]}" 'approval evidence' + parse_file "${cfg[APPROVAL_EVIDENCE_PATH]}" approval 'approval evidence' "$allowed" + local key + for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT; do + [[ -v "approval[$key]" ]] || block "approval evidence is missing $key" + done + [[ ${approval[SCHEMA_VERSION]} == 1 ]] || block 'unsupported approval evidence schema' + for key in ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do + [[ ${approval[$key]} == "${cfg[$key]}" ]] || block "approval evidence does not match exact $key" + done + [[ ${approval[APPROVAL_IDENTITY]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ && ${approval[POLICY_IDENTITY]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ && ${approval[APPROVAL_ID]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ ]] || block 'approval identity is malformed' + valid_utc "${approval[APPROVED_AT]}" || block 'approval timestamp must be UTC RFC3339' + case ${cfg[APPROVAL_PROVIDER]} in + manual-exact-head|external-exact-head) ;; + github-environment) + [[ -v 'cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]' ]] || block 'GitHub Environment approval requires capability evidence' + inside "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" "$etc_root/evidence" || block 'capability evidence is outside the approved evidence directory' + secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' + parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' 'SCHEMA_VERSION ENVIRONMENT TARGET_ID ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' + [[ ${capability[ENVIRONMENT]:-} == "${cfg[ENVIRONMENT]}" && ${capability[TARGET_ID]:-} == "${cfg[TARGET_ID]}" ]] || block 'GitHub Environment capability evidence does not match this installation' + [[ ${capability[SCHEMA_VERSION]:-} == 1 && ${capability[ENVIRONMENT_PROTECTION]:-} == verified && ${capability[EXACT_HEAD]:-} == "${cfg[SOURCE_COMMIT]}" ]] || block 'GitHub Environment capability evidence is not exact-head verified' + if [[ ! ${capability[CAPABILITY_ID]:-} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${capability[CHECKED_AT]:-}"; then block 'GitHub Environment capability evidence is missing identity or UTC time'; fi + ;; + *) block 'unsupported approval provider' ;; + esac + inside "${cfg[CHECKPOINT_EVIDENCE_PATH]}" "$etc_root/evidence" || block 'checkpoint evidence is outside the approved evidence directory' + secure_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" 'checkpoint evidence' + parse_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" checkpoint 'checkpoint evidence' 'SCHEMA_VERSION ENVIRONMENT TARGET_ID CHECKPOINT_ID RECORDED_AT' + [[ ${checkpoint[SCHEMA_VERSION]:-} == 1 && ${checkpoint[ENVIRONMENT]:-} == "$environment" && ${checkpoint[TARGET_ID]:-} == "$target" ]] || block 'checkpoint evidence does not match the explicit environment and target' + if [[ ! ${checkpoint[CHECKPOINT_ID]:-} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${checkpoint[RECORDED_AT]:-}"; then block 'checkpoint evidence is malformed'; fi +} + +require_host() { + local command os_id os_version available required systemd_state disk_path canonical + # Docker checks must target the host's local daemon, not an operator shell's + # remote DOCKER_HOST or selected context. + DOCKER_HOST=unix://$(root_path /run/docker.sock) + export DOCKER_HOST + unset DOCKER_CONTEXT + for command in bash awk base64 cut sort stat sha256sum readlink realpath install cmp mv cp rm mkdir mktemp chmod ln flock kill timeout env git python3 docker systemctl systemd-analyze systemd-inhibit timedatectl curl df date; do + command -v "$command" >/dev/null || block "$command is required" + done + local os_release + os_release=$(root_path /etc/os-release) + if [[ -L "$os_release" ]]; then + canonical=$(readlink -f -- "$os_release") + [[ "$canonical" == "$(root_path /usr/lib/os-release)" ]] || block 'supported Linux os-release metadata is missing' + os_release=$canonical + fi + [[ -f "$os_release" && ! -L "$os_release" ]] || block 'supported Linux os-release metadata is missing' + os_id=$(awk -F= '$1=="ID" {gsub(/"/,"",$2); print $2}' "$os_release") + os_version=$(awk -F= '$1=="VERSION_ID" {gsub(/"/,"",$2); print $2}' "$os_release") + [[ "$os_id" == debian && "$os_version" =~ ^(12|13)(\.|$) || "$os_id" == ubuntu && "$os_version" =~ ^(22\.04|24\.04)$ ]] || block 'unsupported Linux distribution or release' + [[ -d $(root_path /run/systemd/system) ]] || block 'systemd is not the active init system' + systemd_state=$(systemctl is-system-running 2>/dev/null || true) + [[ "$systemd_state" == running || "$systemd_state" == degraded ]] || block 'systemd is unavailable' + docker info >/dev/null 2>&1 || block 'Docker Engine is unavailable' + [[ ${cfg[REQUIRE_COMPOSE]} != 1 ]] || docker compose version >/dev/null 2>&1 || block 'Docker Compose v2 is required but unavailable' + [[ $(timedatectl show -p NTPSynchronized --value 2>/dev/null) == yes ]] || block 'host time is not synchronized' + disk_path=$state_root + while [[ ! -e "$disk_path" ]]; do disk_path=$(dirname "$disk_path"); done + available=$(df -Pk "$disk_path" | awk 'NR==2 {print $4}') + required=$((cfg[MIN_DISK_GIB] * 1024 * 1024)) + ((available >= required)) || block 'insufficient deployer disk capacity' + if [[ "$testing" != 1 || ${CI_FLEET_DEPLOYER_TEST_NETWORK:-} != ok ]]; then + printf '%s\n' "${cfg[NETWORK_HOST]}" | python3 -c 'import socket,sys; socket.getaddrinfo(sys.stdin.readline().strip(), 443)' >/dev/null 2>&1 || block 'network prerequisite DNS lookup failed' + fi + printf 'url = "https://%s/"\nconnect-timeout = 5\nmax-time = 10\nhead\nsilent\n' "${cfg[NETWORK_HOST]}" | curl --config - >/dev/null 2>&1 || block 'network prerequisite HTTPS check failed' + reject_mixed_role +} + +require_maintenance_host() { + # Docker checks must target the host's local daemon, not an operator shell's + # remote DOCKER_HOST or selected context. + DOCKER_HOST=unix://$(root_path /run/docker.sock) + export DOCKER_HOST + unset DOCKER_CONTEXT + local command + for command in bash awk cut stat sha256sum readlink realpath install cp rm mkdir mktemp chmod ln mv flock kill timeout env python3 systemctl systemd-analyze date; do + command -v "$command" >/dev/null || block "$command is required for maintenance" + done + [[ -d "$systemd_root" && ! -L "$systemd_root" ]] || block 'systemd unit directory is unavailable' +} + +reject_mixed_role() { + local unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" runner_unit + for unit in ci-fleet-health.service ci-fleet-health.timer ci-fleet-reconcile.service ci-fleet-reconcile.timer ci-fleet-cleanup.service ci-fleet-cleanup.timer ci-fleet-drift.service ci-fleet-drift.timer actions.runner.service; do + [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" && ! -L "$systemd_root/timers.target.wants/$unit" ]] || block 'ordinary CI controller or runner state is present' + done + shopt -s nullglob + for runner_unit in "$systemd_root"/actions.runner.*.service "$systemd_root"/multi-user.target.wants/actions.runner.*.service; do + shopt -u nullglob + [[ -n "$runner_unit" ]] && block 'ordinary GitHub Actions runner service is present' + done + shopt -u nullglob + for path in "$(root_path /etc/ci-fleet/ci-fleet.env)" "$(root_path /etc/ci-fleet/host.env)" "$(root_path /etc/ci-fleet/secrets)" "$(root_path /opt/ci-fleet/current)" "$(root_path /var/lib/ci-fleet/install-state.json)"; do + [[ ! -e "$path" && ! -L "$path" ]] || block 'ordinary CI controller or runner state is present' + done + output=$(docker ps -a --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || block 'Docker workload inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || block 'unrelated Docker workload is present'; done <<<"$output" + output=$(docker network ls --filter type=custom --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || block 'Docker network inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || block 'incompatible custom Docker network is present'; done <<<"$output" + output=$(docker volume ls --format '{{.Name}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || block 'Docker volume inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || block 'incompatible Docker volume is present'; done <<<"$output" +} + +validate_checkout() { + local head + git_checkout() { git -c core.fsmonitor= -c core.hooksPath=/dev/null -C "$repo_root" "$@"; } + head=$(git_checkout --no-replace-objects rev-parse 'HEAD^{commit}') || block 'installer checkout is not Git-authored' + [[ "$head" == "$core_ref" ]] || block 'CORE_REF must equal the exact reviewed checkout HEAD' + if [[ "$testing" != 1 ]]; then + git_checkout diff --quiet HEAD -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer || block 'reviewed deployer inputs differ from HEAD' + fi + # Pin the reviewed inputs before any privileged copy: copy the worktree + # bytes once into a root-controlled snapshot outside managed state, then + # require each copied file's Git blob identity to equal the pinned commit's + # tree entry. The commit SHA is content-addressed, so no mutation of the + # worktree, refs, or loose objects can substitute bytes under $head. + local path blob tree_listing + checkout_snapshot=$(mktemp -d) + chmod 0700 "$checkout_snapshot" + install -d -m 0700 "$checkout_snapshot/scripts" "$checkout_snapshot/deploy/deployer" + install -m 0755 "$repo_root/scripts/install-deployer.sh" "$repo_root/scripts/deployer-runtime.sh" "$checkout_snapshot/scripts/" + local entry + for entry in "$repo_root"/deploy/deployer/*; do + [[ -f "$entry" && ! -L "$entry" ]] || block 'deployer unit source contains an unsafe or untracked entry' + install -m 0644 "$entry" "$checkout_snapshot/deploy/deployer/" + done + tree_listing=$(git_checkout --no-replace-objects ls-tree -r "$head" -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer) || block 'reviewed commit tree is unreadable' + [[ -n $tree_listing ]] || block 'reviewed commit tree is unreadable' + while read -r _ _ blob path; do + [[ $blob == "$(git hash-object "$checkout_snapshot/$path")" ]] || block "checkout input $path differs from the reviewed commit" + done <<<"$tree_listing" + # The snapshot must contain exactly the reviewed entries: any extra copied + # path (for example an untracked symlink target) is unreviewed content. + local snapshot_listing + snapshot_listing=$(cd "$checkout_snapshot" && find scripts deploy -type f | sort) + [[ $snapshot_listing == "$(awk '{print $4}' <<<"$tree_listing" | sort)" ]] || block 'checkout snapshot contains unreviewed entries' + repo_root=$checkout_snapshot + unit_source=$repo_root/deploy/deployer +} + +active_deployment() { + local pid started now boot_id start_time live_start + [[ -f "$active_operation" && ! -L "$active_operation" ]] || return 1 + pid=$(awk -F= '$1=="pid" {print $2}' "$active_operation") + started=$(awk -F= '$1=="started_at" {print $2}' "$active_operation") + [[ "$pid" =~ ^[1-9][0-9]*$ && "$started" =~ ^[0-9]+$ ]] || return 0 + if kill -0 "$pid" 2>/dev/null; then + if [[ "$testing" == 1 ]]; then + [[ -n ${CI_FLEET_DEPLOYER_TEST_LIVE_PID:-} && $pid == "$CI_FLEET_DEPLOYER_TEST_LIVE_PID" ]] && return 0 + return 1 + fi + boot_id=$(awk -F= '$1=="boot_id" {print $2}' "$active_operation") + start_time=$(awk -F= '$1=="start_time" {print $2}' "$active_operation") + if [[ -n $boot_id && -n $start_time ]]; then + [[ $boot_id == "$(/dev/null) + [[ -n $live_start ]] || return 0 + [[ $live_start == "$start_time" ]] || return 1 + fi + return 0 + fi + now=$(date +%s) + ((started <= now)) || return 0 + ((now - started <= 3600)) || return 1 + return 0 +} + +release_complete() { + local release=$1 stored actual unit entry dir + [[ -d "$release" && ! -L "$release" && $(stat -c '%u:%a' "$release") == "$expected_uid:755" && -x "$release/scripts/install-deployer.sh" && -x "$release/scripts/deployer-runtime.sh" ]] || return 1 + for dir in "$release/scripts" "$release/deploy" "$release/deploy/deployer"; do [[ -d "$dir" && ! -L "$dir" && $(stat -c '%u:%a' "$dir") == "$expected_uid:755" ]] || return 1; done + for entry in "$release/scripts/install-deployer.sh" "$release/scripts/deployer-runtime.sh"; do [[ ! -L "$entry" && $(stat -c '%u:%a' "$entry") == "$expected_uid:755" ]] || return 1; done + [[ -f "$release/.ci-fleet-tree-sha256" ]] || return 1 + stored=$(<"$release/.ci-fleet-tree-sha256") + actual=$(cd "$release" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1) + [[ "$stored" == "$actual" ]] || return 1 + for unit in "${unit_names[@]}"; do [[ -f "$release/deploy/deployer/$unit" && ! -L "$release/deploy/deployer/$unit" && $(stat -c '%u:%a' "$release/deploy/deployer/$unit") == "$expected_uid:644" ]] || return 1; done +} + +state_matches() { + [[ -f "$state_file" && ! -L "$state_file" && $(stat -c '%u:%a' "$state_file") == "$expected_uid:600" ]] || return 1 + [[ -f "$active_policy" && ! -L "$active_policy" && $(stat -c '%u:%a' "$active_policy") == "$expected_uid:600" && $(cmp -s "$config" "$active_policy"; echo $?) == 0 ]] || return 1 + printf '%s\n' "$core_ref" "$environment" "$target" "${cfg[DEPLOYER_IDENTITY]}" "${cfg[SOURCE_COMMIT]}" "$artifact" "${approval[APPROVAL_ID]}" "${approval[APPROVAL_IDENTITY]}" "${approval[POLICY_IDENTITY]}" "${cfg[APPROVAL_PROVIDER]}" "${checkpoint[CHECKPOINT_ID]}" | python3 -c ' +import json, sys +try: value=json.load(open(sys.argv[1], encoding="utf-8")) +except (OSError, ValueError): raise SystemExit(1) +keys=("core_ref","environment","target","deployer_identity","source_commit","artifact","approval_id","approval_identity","policy_identity","approval_provider","checkpoint_id") +expected=[line.rstrip("\n") for line in sys.stdin] +raise SystemExit(0 if all(value.get(k)==v for k,v in zip(keys,expected)) else 1) +' "$state_file" +} + +units_match() { + local unit + for unit in "${unit_names[@]}"; do + [[ -f "$systemd_root/$unit" && ! -L "$systemd_root/$unit" && $(stat -c '%u:%a' "$systemd_root/$unit") == "$expected_uid:644" ]] || return 1 + cmp -s "$unit_source/$unit" "$systemd_root/$unit" || return 1 + [[ ! -e "$systemd_root/$unit.d" && ! -L "$systemd_root/$unit.d" ]] || return 1 + done + for unit in "${timer_names[@]}"; do [[ $(systemctl is-enabled "$unit" 2>/dev/null) == enabled ]] && systemctl is-active "$unit" >/dev/null 2>&1 || return 1; done +} + +current_matches() { + local target_path=$releases/$core_ref + [[ -L "$current" && $(readlink "$current") == "releases/$core_ref" ]] || return 1 + release_complete "$target_path" +} + +managed_boundaries_match() { + local path mode + for path in "$install_root:755" "$releases:755" "$state_root:700" "$lock_root:700" "$log_root:700"; do + mode=${path##*:}; path=${path%:*} + [[ -d "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || return 1 + done + [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || return 1 + mode=$(stat -c %a "$systemd_root") + (((8#$mode & 8#022) == 0)) +} + +converged() { + [[ -e "$deployed_current" || -L "$deployed_current" ]] || return 1 + (load_deployed_snapshot) >/dev/null 2>&1 || return 1 + managed_boundaries_match && current_matches && state_matches && units_match +} + +acquire_lock() { + local path + secure_directory "$lock_root" 700 1 + if [[ -e "$state_root" || -L "$state_root" ]]; then secure_directory "$state_root" 700 0 || block 'managed state boundary is unsafe'; fi + exec 9<"$lock_root" + flock -n 9 || block 'another deployer installer operation is running' + # Role admission is serialized across the controller and deployer installers: + # both flock this shared path for their whole mutating run. + install -d -m 0755 "$(root_path /run)" + exec 8>"$(root_path /run/ci-fleet-role-admission.lock)" + flock -n 8 || block 'another ci-fleet role installation is already running' + [[ ! -L "$active_operation" ]] || block 'active operation marker is an unsafe symlink' + if [[ -e "$active_operation" && ! -f "$active_operation" ]]; then block 'active operation marker has an unsafe type'; fi + if [[ -e "$active_operation" ]] && ! active_deployment; then + [[ ! -L "$active_operation" && -f "$active_operation" && $(stat -c '%u:%a' "$active_operation") == "$expected_uid:600" ]] || block 'stale operation state is unsafe' + rm -f "$active_operation" + fi + if [[ -d "$releases" ]]; then + secure_directory "$install_root" 755 0 + secure_directory "$releases" 755 0 + shopt -s nullglob + for path in "$releases"/."$core_ref".staging.*; do + [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:755" ]] || block 'interrupted release staging state is unsafe' + rm -rf -- "$path" + done + shopt -u nullglob + fi + shopt -s nullglob + for path in "$state_root"/.transaction-preparing.*; do + [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:700" ]] || block 'incomplete transaction preparation is unsafe' + rm -rf -- "$path" + done + shopt -u nullglob + recover_interrupted_transaction +} + +acquire_check_lock() { + [[ -d "$lock_root" && ! -L "$lock_root" && $(stat -c '%u:%a' "$lock_root") == "$expected_uid:700" ]] || block 'installed deployer lock boundary is absent or unsafe' + if [[ -e "$active_operation" || -L "$active_operation" ]]; then + [[ -f "$active_operation" && ! -L "$active_operation" ]] || block 'active operation marker has an unsafe type' + fi + exec 9<"$lock_root" + flock -n 9 || block 'another deployer operation is running' +} + +begin_transaction() { + local name path current_target transaction_name transaction_ready deployed_target enabled_state systemd_mode + [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' + systemd_mode=$(stat -c %a "$systemd_root") + (((8#$systemd_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' + [[ -d "$install_root" && ! -L "$install_root" && $(stat -c '%u:%a' "$install_root") == "$expected_uid:755" ]] || block 'managed install boundary has an unsafe owner, mode, or type' + for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + path=$state_root/$name + [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:600" ]] || block 'managed transaction state has an unsafe type, owner, or mode' + done + if [[ -L "$current" ]]; then + current_target=$(readlink "$current") + [[ "$current_target" =~ ^releases/[0-9a-f]{40}$ ]] || block 'current release pointer is unsafe' + elif [[ -e "$current" ]]; then block 'current release pointer has an unsafe type' + fi + for name in "${unit_names[@]}"; do + path=$systemd_root/$name + [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:644" ]] || block 'managed systemd unit has an unsafe owner, mode, or type' + done + transaction_dir=$(mktemp -d "$state_root/.transaction-preparing.XXXXXX") + transaction_preparing=1 + chmod 0700 "$transaction_dir" + install -d -m 0700 "$transaction_dir/units" "$transaction_dir/state" + for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + path=$state_root/$name + if [[ -f "$path" ]]; then + install -m 0600 "$path" "$transaction_dir/state/$name" + printf '%s\n' "$name" >>"$transaction_dir/state-present" + fi + done + [[ -z ${current_target:-} ]] || printf '%s\n' "$current_target" >"$transaction_dir/current-target" + for name in "${unit_names[@]}"; do + path=$systemd_root/$name + if [[ -f "$path" ]]; then + install -m 0644 "$path" "$transaction_dir/units/$name" + printf '%s\n' "$name" >>"$transaction_dir/units-present" + fi + done + for name in "${timer_names[@]}"; do + enabled_state=$(systemctl is-enabled "$name" 2>/dev/null) || enabled_state= + if [[ $enabled_state == enabled ]]; then printf '%s\n' "$name" >>"$transaction_dir/timers-enabled"; fi + if [[ $enabled_state == enabled-runtime ]]; then printf '%s\n' "$name" >>"$transaction_dir/timers-enabled-runtime"; fi + if systemctl is-active "$name" >/dev/null 2>&1; then printf '%s\n' "$name" >>"$transaction_dir/timers-active"; fi + done + if [[ -L "$deployed_current" ]]; then + deployed_target=$(readlink "$deployed_current") + [[ "$deployed_target" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'deployed snapshot pointer is unsafe' + printf '%s\n' "$deployed_target" >"$transaction_dir/deployed-target" + elif [[ -e "$deployed_current" ]]; then block 'deployed snapshot pointer has an unsafe type' + else printf 'absent\n' >"$transaction_dir/deployed-target" + fi + transaction_name=${transaction_dir##*/} + transaction_ready=$state_root/.transaction.${transaction_name#.transaction-preparing.} + sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || block 'prepared recovery journal is not durable' + mv "$transaction_dir" "$transaction_ready" + transaction_dir=$transaction_ready + sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || block 'published recovery journal is not durable' + transaction_preparing=0 +} + +restore_transaction() { + local name target_value backed_up systemd_restore_mode + [[ -n ${transaction_dir:-} && -d $transaction_dir ]] || return 0 + transaction_committed=1 + for name in units-present state-present timers-enabled timers-enabled-runtime timers-active current-target deployed-target deployed-created; do + [[ ! -e "$transaction_dir/$name" && ! -L "$transaction_dir/$name" ]] || [[ ! -L "$transaction_dir/$name" && -f "$transaction_dir/$name" ]] || block "transaction manifest $name has an unsafe type" + done + if [[ -f "$transaction_dir/units-present" ]]; then + while IFS= read -r name; do + [[ " ${unit_names[*]} " == *" $name "* && -f "$transaction_dir/units/$name" && ! -L "$transaction_dir/units/$name" ]] || block 'transaction unit manifest is unsafe' + done <"$transaction_dir/units-present" + fi + if [[ -f "$transaction_dir/state-present" ]]; then + while IFS= read -r name; do + [[ " install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf " == *" $name "* && -f "$transaction_dir/state/$name" && ! -L "$transaction_dir/state/$name" ]] || block 'transaction state manifest is unsafe' + done <"$transaction_dir/state-present" + fi + for name in timers-enabled timers-enabled-runtime timers-active; do + if [[ -f "$transaction_dir/$name" ]]; then + while IFS= read -r target_value; do + [[ " ${timer_names[*]} " == *" $target_value "* ]] || block 'transaction timer manifest is unsafe' + done <"$transaction_dir/$name" + fi + done + if [[ -f "$transaction_dir/current-target" ]]; then + [[ $(<"$transaction_dir/current-target") =~ ^releases/[0-9a-f]{40}$ ]] || block 'transaction current pointer is unsafe' + fi + if [[ -f "$transaction_dir/deployed-target" ]]; then + target_value=$(<"$transaction_dir/deployed-target") + [[ "$target_value" == absent || "$target_value" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'transaction deployed pointer is unsafe' + fi + for name in units state; do + [[ -d "$transaction_dir/$name" && ! -L "$transaction_dir/$name" ]] || block "transaction $name backup directory is unsafe" + backed_up=$(cd "$transaction_dir/$name" && shopt -s nullglob; printf '%s\n' * | sort) + if [[ -f "$transaction_dir/$name-present" ]]; then + [[ $backed_up == "$(sort "$transaction_dir/$name-present")" ]] || block "transaction $name manifest does not match its backup directory" + else + [[ -z $backed_up ]] || block "transaction $name manifest is missing but backups remain" + fi + done + # Validate the deployed-state boundary before any pointer mutation follows it. + if [[ -e "$deployed_root" || -L "$deployed_root" ]]; then + [[ -d "$deployed_root" && ! -L "$deployed_root" && $(stat -c '%u:%a' "$deployed_root") == "$expected_uid:700" ]] || block 'deployed snapshot directory is unsafe' + fi + [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' + systemd_restore_mode=$(stat -c %a "$systemd_root") + (((8#$systemd_restore_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' + [[ -d "$install_root" && ! -L "$install_root" && $(stat -c '%u:%a' "$install_root") == "$expected_uid:755" ]] || block 'managed install boundary has an unsafe owner, mode, or type' + for name in "${timer_names[@]}"; do + if [[ -e "$systemd_root/$name" || -L "$systemd_root/$name" ]] || systemctl is-enabled "$name" >/dev/null 2>&1 || systemctl is-active "$name" >/dev/null 2>&1; then + systemctl disable --now "$name" >/dev/null 2>&1 || return + fi + done + for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name" || return; done + if [[ -f "$transaction_dir/units-present" ]]; then + while IFS= read -r name; do + install -m 0644 "$transaction_dir/units/$name" "$systemd_root/$name" || return + done <"$transaction_dir/units-present" + fi + for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do rm -f -- "$state_root/$name" || return; done + if [[ -f "$transaction_dir/state-present" ]]; then + while IFS= read -r name; do + install -m 0600 "$transaction_dir/state/$name" "$state_root/$name" || return + done <"$transaction_dir/state-present" + fi + rm -f -- "$current" "$install_root/.current.new" "$state_root/.install-state.new" "$active_policy.new" "$state_file.new" || return + if [[ -f "$transaction_dir/current-target" ]]; then + target_value=$(<"$transaction_dir/current-target") + ln -s "$target_value" "$current" || return + fi + if [[ -f "$transaction_dir/deployed-target" ]]; then + target_value=$(<"$transaction_dir/deployed-target") + if [[ "$target_value" == absent ]]; then + rm -f -- "$deployed_current" || return + else + rm -f -- "$deployed_current" || return + ln -s "$target_value" "$deployed_current" || return + fi + fi + if [[ -f "$transaction_dir/deployed-created" ]]; then + target_value=$(<"$transaction_dir/deployed-created") + [[ $target_value =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'transaction created snapshot name is unsafe' + if [[ ! -e $deployed_current && ! -L $deployed_current ]] || [[ $(readlink "$deployed_current" 2>/dev/null) != "$target_value" ]]; then + [[ ! -d "$deployed_root/$target_value" || -L "$deployed_root/$target_value" ]] || rm -rf -- "${deployed_root:?}/$target_value" + fi + fi + systemctl daemon-reload >/dev/null 2>&1 || return + if [[ -f "$transaction_dir/timers-enabled" ]]; then + while IFS= read -r name; do + systemctl enable "$name" >/dev/null 2>&1 || return + done <"$transaction_dir/timers-enabled" + fi + if [[ -f "$transaction_dir/timers-enabled-runtime" ]]; then + while IFS= read -r name; do + systemctl enable --runtime "$name" >/dev/null 2>&1 || return + done <"$transaction_dir/timers-enabled-runtime" + fi + if [[ -f "$transaction_dir/timers-active" ]]; then + while IFS= read -r name; do + systemctl start "$name" >/dev/null 2>&1 || return + done <"$transaction_dir/timers-active" + fi + # Restored state, pointer, and units may live on separate filesystems; make + # them durable before the only recovery journal is deleted, then persist the + # journal retirement itself. + sync -f "$state_root" 2>/dev/null || block 'restored host state is not durable' + sync -f "$install_root" 2>/dev/null || block 'restored install root is not durable' + sync -f "$systemd_root" 2>/dev/null || block 'restored systemd boundary is not durable' + rm -rf -- "$transaction_dir" || return + transaction_dir= + sync -f "$state_root" 2>/dev/null || block 'retired recovery journal is not durable' +} + +recover_interrupted_transaction() { + local candidates=() candidate + [[ -d "$state_root" ]] || return 0 + shopt -s nullglob + candidates=("$state_root"/.transaction.*) + shopt -u nullglob + ((${#candidates[@]} <= 1)) || block 'multiple interrupted installer transactions require operator recovery' + ((${#candidates[@]} == 1)) || return 0 + candidate=${candidates[0]} + [[ ! -L "$candidate" && -d "$candidate" && $(stat -c '%u:%a' "$candidate") == "$expected_uid:700" ]] || block 'interrupted installer transaction is unsafe' + transaction_dir=$candidate + if [[ -e "$transaction_dir/application-rollback-committed" || -L "$transaction_dir/application-rollback-committed" ]]; then + finalize_committed_rollback + recovered_rollback=1 + transaction_committed=0 + return + fi + # Restoration does not consume deployed-snapshot dependencies; requiring + # them here would wedge recovery behind unrelated deployed drift. + restore_transaction + transaction_committed=0 +} + +finalize_committed_rollback() { + local marker=$transaction_dir/application-rollback-committed retired + local final_adapter_path final_adapter_sha final_credential_provider final_credential_ref + [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' + secure_file "$active_policy" 'active policy' || return + secure_file "$state_file" 'deployer install state' || return + final_adapter_path=$(awk '$0 ~ /^ADAPTER_PATH=/ {sub(/^ADAPTER_PATH=/, ""); print}' "$active_policy") + final_adapter_sha=$(awk '$0 ~ /^ADAPTER_SHA256=/ {sub(/^ADAPTER_SHA256=/, ""); print}' "$active_policy") + inside "$final_adapter_path" "$etc_root/adapters" || return + [[ ! -L "$final_adapter_path" && -f "$final_adapter_path" && $(stat -c '%u:%a' "$final_adapter_path") == "$expected_uid:700" ]] || return + [[ $(sha256sum "$final_adapter_path" | cut -d' ' -f1) == "$final_adapter_sha" ]] || return + final_credential_provider=$(awk '$0 ~ /^CREDENTIAL_PROVIDER=/ {sub(/^CREDENTIAL_PROVIDER=/, ""); print}' "$active_policy") + final_credential_ref=$(awk '$0 ~ /^CREDENTIAL_REF=/ {sub(/^CREDENTIAL_REF=/, ""); print}' "$active_policy") + credential_reference_safe "$final_credential_provider" "$final_credential_ref" 'committed rollback policy' || return + python3 - "$state_file" "$active_policy" "$etc_root/adapters" "$etc_root/credentials" <<'PY' >/dev/null 2>&1 || return +import json, re, sys +try: + state = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): + raise SystemExit(1) +policy = {} +allowed = {"SCHEMA_VERSION","CORE_REF","ENVIRONMENT","TARGET_ID","DEPLOYER_IDENTITY","ADAPTER_PATH","ADAPTER_SHA256","CREDENTIAL_PROVIDER","CREDENTIAL_REF","CREDENTIAL_SCOPE","APPROVAL_PROVIDER","APPROVAL_EVIDENCE_PATH","APPROVAL_CAPABILITY_EVIDENCE_PATH","PRODUCTION_AUTHORIZATION_EVIDENCE_PATH","CHECKPOINT_EVIDENCE_PATH","SOURCE_COMMIT","ARTIFACT_IMAGE","NETWORK_HOST","MIN_DISK_GIB","REQUIRE_COMPOSE"} +try: + for line in open(sys.argv[2], encoding='utf-8'): + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + key, sep, value = line.partition('=') + if not sep or key in policy or key not in allowed: + raise SystemExit(1) + policy[key] = value +except OSError: + raise SystemExit(1) +sha = re.compile(r'^[0-9a-f]{40}$') +pairs = (('core_ref','CORE_REF'), ('environment','ENVIRONMENT'), ('target','TARGET_ID'), ('source_commit','SOURCE_COMMIT'), ('artifact','ARTIFACT_IMAGE'), ('deployer_identity','DEPLOYER_IDENTITY')) +ok = (policy.get('SCHEMA_VERSION') == '1' + and all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) + and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) + and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact']))) +ok = ok and all(policy.get(k) for k in ('ADAPTER_PATH','ADAPTER_SHA256','CREDENTIAL_PROVIDER','CREDENTIAL_REF','CREDENTIAL_SCOPE')) +ok = ok and bool(re.fullmatch(r'[0-9a-f]{64}', policy.get('ADAPTER_SHA256',''))) +ok = ok and policy.get('ADAPTER_PATH','').startswith(sys.argv[3] + '/') +ok = ok and policy.get('CREDENTIAL_PROVIDER') in ('file','external') +ok = ok and policy.get('CREDENTIAL_SCOPE') == policy.get('ENVIRONMENT') +if policy.get('CREDENTIAL_PROVIDER') == 'file': + ok = ok and policy.get('CREDENTIAL_REF','').startswith(sys.argv[4] + '/') +else: + ok = ok and bool(re.fullmatch(r'external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}', policy.get('CREDENTIAL_REF',''))) +raise SystemExit(0 if ok else 1) +PY + publish_deployed_snapshot "$active_policy" "$state_file" || return + rm -f "$previous_state" "$previous_policy" || return + retired=$state_root/.retired.$$.transaction + mv -Tf "$transaction_dir" "$retired" || return + transaction_dir= + transaction_committed=1 + rm -rf -- "$retired" + # The journal rename and deletion must persist with the finalization, or a + # reboot can resurrect the committed-rollback transaction and repeat it. + sync -f "$state_root" 2>/dev/null || block 'retired recovery journal is not durable' +} + +commit_transaction() { + local retired + sync -f "$state_file" "$active_policy" 2>/dev/null || block 'committed host state is not durable' + sync -f "$state_root" 2>/dev/null || block 'committed host state is not durable' + sync -f "$install_root" 2>/dev/null || block 'committed install root is not durable' + sync -f "$systemd_root" 2>/dev/null || block 'committed systemd boundary is not durable' + retired=$state_root/.retired.$$.transaction + mv -Tf "$transaction_dir" "$retired" || return + transaction_dir= + transaction_committed=1 + rm -rf -- "$retired" + # The journal rename and deletion must persist with the commit, or a reboot + # can resurrect the pre-operation transaction and restore stale state. + sync -f "$state_root" 2>/dev/null || block 'retired recovery journal is not durable' +} + +atomic_replace_directory() { + local replacement=$1 target_path=$2 + if [[ ! -e "$target_path" && ! -L "$target_path" ]]; then mv "$replacement" "$target_path"; return; fi + [[ ! -L "$target_path" && -d "$target_path" ]] || block 'managed release target has an unsafe type' + python3 - "$replacement" "$target_path" <<'PY' +import ctypes, os, sys +source, target = map(os.fsencode, sys.argv[1:]) +libc = ctypes.CDLL(None, use_errno=True) +renameat2 = getattr(libc, 'renameat2', None) +if renameat2 is None: + raise OSError('atomic directory exchange is unavailable') +renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] +if renameat2(-100, source, -100, target, 2) != 0: + error = ctypes.get_errno(); raise OSError(error, os.strerror(error)) +fd = os.open(os.path.dirname(target), os.O_RDONLY | os.O_DIRECTORY) +try: os.fsync(fd) +finally: os.close(fd) +PY +} + +install_release() { + local release=$releases/$core_ref staging + secure_directory "$install_root" 755 1 + secure_directory "$releases" 755 1 + if release_complete "$release"; then return; fi + staging=$(mktemp -d "$releases/.${core_ref}.staging.XXXXXX") + chmod 0755 "$staging" + staging_path=$staging + install -d -m 0755 "$staging/scripts" "$staging/deploy/deployer" + install -m 0755 "$repo_root/scripts/install-deployer.sh" "$repo_root/scripts/deployer-runtime.sh" "$staging/scripts/" + install -m 0644 "$unit_source"/* "$staging/deploy/deployer/" + (cd "$staging" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1) >"$staging/.ci-fleet-tree-sha256" + chmod 0644 "$staging/.ci-fleet-tree-sha256" + release_complete "$staging" || die 'staged deployer release is incomplete' + atomic_replace_directory "$staging" "$release" + [[ ! -e "$staging" ]] || rm -rf -- "$staging" + staging_path= +} + +write_state() { + local destination=$1 temporary + temporary=$(mktemp "$state_root/.state.XXXXXX") + printf '%s\n' "$core_ref" "$environment" "$target" "${cfg[DEPLOYER_IDENTITY]}" "${cfg[SOURCE_COMMIT]}" "$artifact" "${approval[APPROVAL_ID]}" "${approval[APPROVAL_IDENTITY]}" "${approval[POLICY_IDENTITY]}" "${cfg[APPROVAL_PROVIDER]}" "${checkpoint[CHECKPOINT_ID]}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" | python3 -c ' +import json,sys +keys=("core_ref","environment","target","deployer_identity","source_commit","artifact","approval_id","approval_identity","policy_identity","approval_provider","checkpoint_id","installed_at") +values=[line.rstrip("\n") for line in sys.stdin] +with open(sys.argv[1],"w",encoding="utf-8") as f: json.dump(dict(zip(keys,values)),f,indent=2,sort_keys=True); f.write("\n") +' "$temporary" + chmod 0600 "$temporary" + mv -Tf "$temporary" "$destination" +} + +load_deployed_snapshot() { + local snapshot deployed_adapter_path deployed_adapter_sha + local deployed_credential_provider deployed_credential_ref deployed_credential_scope deployed_environment + secure_directory "$deployed_root" 700 0 || block 'deployed snapshot directory is missing' + [[ -L "$deployed_current" ]] || block 'deployed snapshot pointer is absent or unsafe' + [[ $(readlink "$deployed_current") =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'deployed snapshot pointer target is not canonical' + snapshot=$(readlink -f "$deployed_current") + inside "$snapshot" "$deployed_root" || block 'deployed snapshot pointer escapes managed state' + secure_directory "$snapshot" 700 0 || block 'deployed snapshot is unsafe' + secure_file "$snapshot/policy.conf" 'deployed rollback policy' + secure_file "$snapshot/state.json" 'deployed rollback state' + deployed_snapshot_policy=$snapshot/policy.conf + deployed_snapshot_state=$snapshot/state.json + deployed_adapter_path=$(awk '$0 ~ /^ADAPTER_PATH=/ {sub(/^ADAPTER_PATH=/, ""); print}' "$deployed_snapshot_policy") + deployed_adapter_sha=$(awk '$0 ~ /^ADAPTER_SHA256=/ {sub(/^ADAPTER_SHA256=/, ""); print}' "$deployed_snapshot_policy") + inside "$deployed_adapter_path" "$etc_root/adapters" || block 'deployed rollback adapter is outside the protected adapter directory' + [[ ! -L "$deployed_adapter_path" && -f "$deployed_adapter_path" && $(stat -c '%u:%a' "$deployed_adapter_path") == "$expected_uid:700" ]] || block 'deployed rollback adapter is missing or unsafe' + [[ $(sha256sum "$deployed_adapter_path" | cut -d' ' -f1) == "$deployed_adapter_sha" ]] || block 'deployed rollback adapter digest does not match its snapshot policy' + deployed_credential_provider=$(awk '$0 ~ /^CREDENTIAL_PROVIDER=/ {sub(/^CREDENTIAL_PROVIDER=/, ""); print}' "$deployed_snapshot_policy") + deployed_credential_ref=$(awk '$0 ~ /^CREDENTIAL_REF=/ {sub(/^CREDENTIAL_REF=/, ""); print}' "$deployed_snapshot_policy") + credential_reference_safe "$deployed_credential_provider" "$deployed_credential_ref" 'deployed rollback policy' + deployed_credential_scope=$(awk '$0 ~ /^CREDENTIAL_SCOPE=/ {sub(/^CREDENTIAL_SCOPE=/, ""); print}' "$deployed_snapshot_policy") + deployed_environment=$(awk '$0 ~ /^ENVIRONMENT=/ {sub(/^ENVIRONMENT=/, ""); print}' "$deployed_snapshot_policy") + [[ $deployed_credential_scope == "$deployed_environment" ]] || block 'deployed rollback credential scope does not match its environment' + python3 - "$deployed_snapshot_state" "$deployed_snapshot_policy" "$etc_root/adapters" "$etc_root/credentials" <<'PY' >/dev/null 2>&1 || block 'deployed rollback snapshot state and policy do not cross-validate' +import json, re, sys +try: + state = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): + raise SystemExit(1) +policy = {} +allowed = {"SCHEMA_VERSION","CORE_REF","ENVIRONMENT","TARGET_ID","DEPLOYER_IDENTITY","ADAPTER_PATH","ADAPTER_SHA256","CREDENTIAL_PROVIDER","CREDENTIAL_REF","CREDENTIAL_SCOPE","APPROVAL_PROVIDER","APPROVAL_EVIDENCE_PATH","APPROVAL_CAPABILITY_EVIDENCE_PATH","PRODUCTION_AUTHORIZATION_EVIDENCE_PATH","CHECKPOINT_EVIDENCE_PATH","SOURCE_COMMIT","ARTIFACT_IMAGE","NETWORK_HOST","MIN_DISK_GIB","REQUIRE_COMPOSE"} +try: + for line in open(sys.argv[2], encoding='utf-8'): + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + key, sep, value = line.partition('=') + if not sep or key in policy or key not in allowed: + raise SystemExit(1) + policy[key] = value +except OSError: + raise SystemExit(1) +sha = re.compile(r'^[0-9a-f]{40}$') +pairs = (('core_ref','CORE_REF'), ('environment','ENVIRONMENT'), ('target','TARGET_ID'), ('source_commit','SOURCE_COMMIT'), ('artifact','ARTIFACT_IMAGE'), ('deployer_identity','DEPLOYER_IDENTITY')) +ok = (policy.get('SCHEMA_VERSION') == '1' + and all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) + and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) + and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact']))) +ok = ok and all(policy.get(k) for k in ('ADAPTER_PATH','ADAPTER_SHA256','CREDENTIAL_PROVIDER','CREDENTIAL_REF','CREDENTIAL_SCOPE')) +ok = ok and bool(re.fullmatch(r'[0-9a-f]{64}', policy.get('ADAPTER_SHA256',''))) +ok = ok and policy.get('ADAPTER_PATH','').startswith(sys.argv[3] + '/') +ok = ok and policy.get('CREDENTIAL_PROVIDER') in ('file','external') +ok = ok and policy.get('CREDENTIAL_SCOPE') == policy.get('ENVIRONMENT') +if policy.get('CREDENTIAL_PROVIDER') == 'file': + ok = ok and policy.get('CREDENTIAL_REF','').startswith(sys.argv[4] + '/') +else: + ok = ok and bool(re.fullmatch(r'external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}', policy.get('CREDENTIAL_REF',''))) +raise SystemExit(0 if ok else 1) +PY +} + +publish_deployed_snapshot() { + local policy=$1 state=$2 snapshot pointer retired incumbent + secure_directory "$deployed_root" 700 1 || return + if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then + # An unusable incumbent snapshot must not block publication of a freshly + # validated rollback pair; only identical bytes short-circuit. + if [[ -L "$deployed_current" ]]; then + incumbent=$(readlink -f "$deployed_current") + if [[ $incumbent == "$deployed_root"/.snapshot.* && ! -L "$incumbent" && -d "$incumbent" && $(stat -c '%u:%a' "$incumbent") == "$expected_uid:700" && -f "$incumbent/policy.conf" && ! -L "$incumbent/policy.conf" && $(stat -c '%u:%a' "$incumbent/policy.conf") == "$expected_uid:600" && -f "$incumbent/state.json" && ! -L "$incumbent/state.json" && $(stat -c '%u:%a' "$incumbent/state.json") == "$expected_uid:600" ]] && cmp -s "$incumbent/policy.conf" "$policy" && cmp -s "$incumbent/state.json" "$state"; then return; fi + fi + retired=$(readlink "$deployed_current") + [[ "$retired" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'current deployed snapshot pointer is unsafe' + rm -f -- "$deployed_current" || return + fi + snapshot=$(mktemp -d "$deployed_root/.snapshot.XXXXXX") || return + chmod 0700 "$snapshot" || return + install -m 0600 "$policy" "$snapshot/policy.conf" || return + install -m 0600 "$state" "$snapshot/state.json" || return + pointer=$(mktemp -u "$deployed_root/.current.XXXXXX") || return + ln -s "${snapshot##*/}" "$pointer" || return + sync -f "$snapshot/policy.conf" "$snapshot/state.json" 2>/dev/null || block 'replacement deployed snapshot is not durable' + sync -f "$snapshot" 2>/dev/null || block 'replacement deployed snapshot is not durable' + mv -Tf "$pointer" "$deployed_current" || return + sync -f "$deployed_root" 2>/dev/null || block 'deployed snapshot pointer is not durable' + if [[ -n ${retired:-} && -d "$deployed_root/$retired" && ! -L "$deployed_root/$retired" ]]; then rm -rf -- "${deployed_root:?}/$retired"; fi +} + +install_units() { + local unit systemd_mode + [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' + systemd_mode=$(stat -c %a "$systemd_root") + (((8#$systemd_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' + for unit in "${unit_names[@]}"; do + [[ ! -e "$systemd_root/$unit.d" && ! -L "$systemd_root/$unit.d" ]] || block "managed unit $unit has an unreviewed drop-in override" + if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then + [[ ! -L "$systemd_root/$unit" && -f "$systemd_root/$unit" && $(stat -c %u "$systemd_root/$unit") == "$expected_uid" ]] || block 'managed systemd unit has an unsafe owner or type' + fi + install -m 0644 "$unit_source/$unit" "$systemd_root/$unit" + done + systemd-analyze verify "${unit_names[@]/#/$systemd_root/}" >/dev/null || die 'systemd unit verification failed' + systemctl daemon-reload + systemctl enable --now "${timer_names[@]}" >/dev/null +} + +adapter_deadline() { + local operation_name=$1 seconds=120 + [[ "$operation_name" != rollback ]] || seconds=2700 + if [[ "$testing" == 1 && -n ${CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS:-} ]]; then + [[ ${CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS} =~ ^[1-9][0-9]?$ ]] || die 'invalid test-only adapter timeout' + seconds=$CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS + fi + printf '%s' "$seconds" +} + +run_adapter() { + local policy=$1 adapter_path=$2 operation_name=$3 marker=${4:-} seconds + seconds=$(adapter_deadline "$operation_name") + if [[ -n "$marker" && $testing != 1 ]]; then + # The transactional adapter call runs under a shutdown inhibitor, like the + # deploy runtime, so a normal shutdown cannot strand a partially applied + # rollback between core mutation and commit-marker publication. + timeout --signal=TERM --kill-after=10s "${seconds}s" systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ + --why='transactional rollback is active' -- env CI_FLEET_DEPLOYER_CONFIG="$policy" CI_FLEET_DEPLOYER_ROLLBACK_COMMIT="$marker" "$adapter_path" "$operation_name" + return + fi + if [[ -n "$marker" ]]; then + timeout --signal=TERM --kill-after=10s "${seconds}s" env CI_FLEET_DEPLOYER_CONFIG="$policy" CI_FLEET_DEPLOYER_ROLLBACK_COMMIT="$marker" "$adapter_path" "$operation_name" + else + timeout --signal=TERM --kill-after=10s "${seconds}s" env CI_FLEET_DEPLOYER_CONFIG="$policy" "$adapter_path" "$operation_name" + fi +} + +run_verified_adapter() { + local policy=$1 path=$2 digest=$3 operation_name=$4 marker=${5:-} + secure_file "$path" "$operation_name adapter" 700 + exec 7<"$path" + [[ $(sha256sum /proc/$$/fd/7 | cut -d' ' -f1) == "$digest" ]] || die "$operation_name adapter digest mismatch" + run_adapter "$policy" "/proc/$$/fd/7" "$operation_name" "$marker" +} + +policy_adapter_operation() { + local policy=$1 operation_name=$2 description=$3 marker=${4:-} snapshot=${5:-1} key + local -A policy_cfg=() + secure_file "$policy" "$description" + reject_mixed_role + if [[ "$snapshot" == 1 ]]; then + snapshot=$(mktemp "$state_root/.policy-check.XXXXXX") + policy_check_snapshot=$snapshot + install -m 0600 "$policy" "$snapshot" + policy=$snapshot + fi + parse_file "$policy" policy_cfg "$description" "$config_keys" + [[ ${policy_cfg[SCHEMA_VERSION]:-} == 1 ]] || die "$description has an unsupported or missing schema version" + for key in ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE ENVIRONMENT; do [[ -v "policy_cfg[$key]" ]] || die "$description is missing $key"; done + [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" + inside "${policy_cfg[ADAPTER_PATH]}" "$etc_root/adapters" || die "$description adapter path is outside the protected adapter directory" + [[ ${policy_cfg[CREDENTIAL_SCOPE]} == "${policy_cfg[ENVIRONMENT]}" ]] || die "$description credential scope does not match its environment" + credential_reference_safe "${policy_cfg[CREDENTIAL_PROVIDER]}" "${policy_cfg[CREDENTIAL_REF]}" "$description" + local status + run_verified_adapter "$policy" "${policy_cfg[ADAPTER_PATH]}" "${policy_cfg[ADAPTER_SHA256]}" "$operation_name" "$marker" >/dev/null + status=$? + if [[ "$snapshot" != 0 ]]; then rm -f "$snapshot"; policy_check_snapshot=; fi + return "$status" +} + +perform_check() { + local transactions=() + shopt -s nullglob + transactions=("$state_root"/.transaction.*) + shopt -u nullglob + ((${#transactions[@]} == 0)) || block 'interrupted installer transaction requires recovery' + if active_deployment; then block 'active deployment prevents a consistent check'; fi + if [[ -e "$active_operation" || -L "$active_operation" ]]; then block 'stale operation marker requires a mutating recovery'; fi + converged || block 'installed deployer state is absent or drifted' + policy_adapter_operation "$active_policy" health 'active policy' '' 0 || block 'active deployer health check failed' + health=healthy + report NO_CHANGE no none "$(rollback_available)" +} + +perform_converge() { + local had_state=0 old_environment old_target candidate_changed=1 old_release + [[ -f "$state_file" ]] && had_state=1 + if ((!had_state)) && { [[ -L "$current" ]] || compgen -G "$systemd_root/ci-fleet-deployer*" >/dev/null || [[ -e "$active_policy" || -L "$active_policy" || -e "$deployed_current" || -L "$deployed_current" ]]; }; then + block 'installed deployer state is absent or drifted; restore install state before convergence' + fi + if ((had_state)); then + read -r old_environment old_target < <(python3 - "$state_file" <<'PY' +import json,sys +v=json.load(open(sys.argv[1])); print(v.get('environment',''),v.get('target','')) +PY +) + [[ "$old_environment" == "$environment" && "$old_target" == "$target" ]] || block 'installed environment and target identity cannot change in place' + old_deployer_identity=$(python3 - "$state_file" <<'PY' +import json,sys +print(json.load(open(sys.argv[1])).get('deployer_identity','')) +PY +) + [[ -z "$old_deployer_identity" || "$old_deployer_identity" == "${cfg[DEPLOYER_IDENTITY]}" ]] || block 'installed deployer ownership identity cannot change in place' + [[ -e "$deployed_current" || -L "$deployed_current" ]] || block 'deployed rollback snapshot is missing; restore it before convergence' + load_deployed_snapshot + if [[ "$mode" == install && -L "$current" ]] && ! state_matches; then block 'install cannot select a new candidate; use --upgrade or --repair'; fi + elif [[ "$mode" == upgrade ]]; then + block '--upgrade requires an existing installation' + fi + if state_matches; then candidate_changed=0; fi + if [[ "$mode" == upgrade && ! -L "$current" ]]; then block '--upgrade requires an active installation; use --install after uninstall'; fi + if active_deployment; then block 'active deployment prevents this operation'; fi + if converged; then + policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer health check failed' + health=healthy; report NO_CHANGE no none "$(rollback_available)"; return + fi + if [[ -L "$current" ]]; then + old_release=$(readlink -f "$current") + if [[ "$mode" == repair ]]; then + # Repair must not trust or execute a damaged old release; the validated + # checkout replaces it transactionally below. Upgrade and install still + # require a complete active release. + release_complete "$old_release" || true + else + release_complete "$old_release" || block 'active deployer release is incomplete' + policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement' + fi + fi + reject_mixed_role + credential_reference_safe "${cfg[CREDENTIAL_PROVIDER]}" "${cfg[CREDENTIAL_REF]}" 'candidate policy' + run_verified_adapter "$config" "${cfg[ADAPTER_PATH]}" "${cfg[ADAPTER_SHA256]}" validate >/dev/null || die 'candidate adapter validation failed' + install_release + secure_directory "$state_root" 700 1 + secure_directory "$log_root" 700 1 + begin_transaction + if ((had_state && candidate_changed)); then + if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then + load_deployed_snapshot + install -m 0600 "$deployed_snapshot_state" "$previous_state" + install -m 0600 "$deployed_snapshot_policy" "$previous_policy" + fi + fi + ln -sfn "releases/$core_ref" "$install_root/.current.new" + # Publish the candidate activation pointer before unit verification so the + # units' /opt/ci-fleet-deployer/current/... ExecStart paths resolve on a + # fresh install; transaction recovery restores the prior pointer. + mv -Tf "$install_root/.current.new" "$current" + install_units + install -m 0600 "$config" "$active_policy.new" + write_state "$state_root/.install-state.new" + reject_mixed_role + mv -Tf "$active_policy.new" "$active_policy" + mv -Tf "$state_root/.install-state.new" "$state_file" + policy_adapter_operation "$active_policy" health 'candidate policy' || die 'candidate health check failed after activation' + if [[ ! -e "$deployed_current" && ! -L "$deployed_current" ]]; then + publish_deployed_snapshot "$active_policy" "$state_file" + if [[ -n ${transaction_dir:-} && -d $transaction_dir ]]; then readlink "$deployed_current" >"$transaction_dir/deployed-created"; fi + fi + commit_transaction + health=healthy + report CHANGED yes run-check "$(rollback_available)" +} + +perform_rollback() { + local source_commit deployer_identity key + local -A rollback_policy=() + if ((recovered_rollback)); then health=healthy; report CHANGED yes restore-host-policy-evidence-then-check no; return; fi + active_deployment && block 'active deployment prevents rollback' + [[ -f "$previous_state" && -f "$previous_policy" ]] || block 'no last-known-good release is available' + secure_directory "$state_root" 700 0 + secure_file "$previous_state" 'last-known-good state' + secure_file "$previous_policy" 'last-known-good policy' + parse_file "$previous_policy" rollback_policy 'last-known-good policy' "$config_keys" + for key in CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "rollback_policy[$key]" ]] || block "last-known-good policy is missing $key"; done + [[ ${rollback_policy[SCHEMA_VERSION]:-} == 1 ]] || block 'last-known-good policy has an unsupported or missing schema version' + read -r core_ref environment target source_commit artifact deployer_identity < <(python3 - "$previous_state" <<'PY' +import json, sys +try: value=json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): raise SystemExit(1) +print(*(value.get(k, '') for k in ('core_ref','environment','target','source_commit','artifact','deployer_identity'))) +PY + ) || block 'last-known-good state is malformed' + [[ "$core_ref" =~ ^[0-9a-f]{40}$ && "$source_commit" =~ ^[0-9a-f]{40}$ && "$artifact" =~ @sha256:[0-9a-f]{64}$ ]] || block 'last-known-good state has unsafe immutable identifiers' + [[ "$core_ref" == "${rollback_policy[CORE_REF]}" && "$environment" == "${rollback_policy[ENVIRONMENT]}" && "$target" == "${rollback_policy[TARGET_ID]}" && "$source_commit" == "${rollback_policy[SOURCE_COMMIT]}" && "$artifact" == "${rollback_policy[ARTIFACT_IMAGE]}" && "$deployer_identity" == "${rollback_policy[DEPLOYER_IDENTITY]}" ]] || block 'last-known-good state and policy do not match' + [[ "$environment" =~ ^[a-z][a-z0-9-]{0,31}$ && "$target" =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'last-known-good identity is malformed' + # Production deployment paths, including application rollback, remain + # separately gated; no accepted decision enables them yet. + [[ $environment != production ]] || block 'production rollback is not authorized by the current accepted scope' + # A deployed snapshot published at install time describes a candidate that + # was never deployed; rollback is meaningful only after a deployment has + # completed (the runtime records last-request.conf on success). + [[ -f "$state_root/last-request.conf" && ! -L "$state_root/last-request.conf" ]] || block 'no completed deployment is available to roll back to' + cfg[DEPLOYER_IDENTITY]=${rollback_policy[DEPLOYER_IDENTITY]} + command -v docker >/dev/null || block 'docker is required for rollback isolation validation' + reject_mixed_role + begin_transaction + install -m 0600 "$previous_state" "$state_file.new" + install -m 0600 "$previous_policy" "$active_policy.new" + + release_complete "$releases/$core_ref" || die 'last-known-good release is incomplete' + unit_source=$releases/$core_ref/deploy/deployer + ln -sfn "releases/$core_ref" "$install_root/.current.new" + # Publish before unit verification so the units' current/... paths resolve; + # transaction recovery restores the prior pointer. + mv -Tf "$install_root/.current.new" "$current" + install_units + mv -Tf "$active_policy.new" "$active_policy" + mv -Tf "$state_file.new" "$state_file" + reject_mixed_role + # The adapter publishes the authoritative commit marker; the rolled-back + # core (pointer, units, policy, state) must be durable before that call, or + # a power loss could leave the marker durable over reverted core state. + sync -f "$state_file" "$active_policy" 2>/dev/null || die 'rolled-back host state is not durable' + sync -f "$state_root" 2>/dev/null || die 'rolled-back host state is not durable' + sync -f "$install_root" 2>/dev/null || die 'rolled-back install root is not durable' + sync -f "$systemd_root" 2>/dev/null || die 'rolled-back systemd boundary is not durable' + # Run the adapter call under a shutdown inhibitor, like the deploy runtime, + # so a normal shutdown cannot strand a partially applied adapter rollback + # between core mutation and commit-marker publication. + if ! policy_adapter_operation "$active_policy" rollback 'last-known-good policy' "$transaction_dir/application-rollback-committed" && [[ ! -f "$transaction_dir/application-rollback-committed" || -L "$transaction_dir/application-rollback-committed" ]]; then + die 'application adapter rollback failed' + fi + # The rollback already replaced the activation pointer and units on possibly + # separate filesystems; the rolled-back core must be durable before the + # commit marker, or recovery could treat the marker as authoritative while + # the pointer, policy, or units reverted. + sync -f "$install_root" 2>/dev/null || die 'rolled-back install root is not durable' + sync -f "$systemd_root" 2>/dev/null || die 'rolled-back systemd boundary is not durable' + # The authoritative commit marker (on the state filesystem) becomes durable + # only after the /opt and /etc boundaries, so recovery can never observe it + # over reverted core state. + sync -f "$transaction_dir/application-rollback-committed" 2>/dev/null || sync "$transaction_dir/application-rollback-committed" 2>/dev/null || die 'application rollback commit marker is not durable' + sync -f "$state_file" "$active_policy" 2>/dev/null || die 'rolled-back host state is not durable' + sync -f "$state_root" 2>/dev/null || die 'rolled-back host state is not durable' + sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || die 'application rollback commit marker is not durable' + finalize_committed_rollback + recovered_rollback=0 + health=healthy + report CHANGED yes restore-host-policy-evidence-then-check no +} + +perform_drain() { + if active_deployment; then block 'active deployment prevents drain'; fi + secure_directory "$state_root" 700 1 + if [[ -e "$drained" || -L "$drained" ]]; then + secure_file "$drained" 'drain marker' + report NO_CHANGE no safe-to-maintain "$(rollback_available)"; return + fi + temporary=$(mktemp "$state_root/.drained.XXXXXX") + chmod 0600 "$temporary" + mv -Tf "$temporary" "$drained" + sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || block 'drain marker is not durable' + sync -f "$state_root" 2>/dev/null || block 'drain marker publication is not durable' + report CHANGED yes safe-to-maintain "$(rollback_available)" +} + +perform_resume() { + secure_directory "$state_root" 700 0 || block 'deployer state directory is missing' + if active_deployment; then block 'active deployment prevents resume'; fi + local was_drained=0 + if [[ -e "$drained" || -L "$drained" ]]; then secure_file "$drained" 'drain marker'; was_drained=1; fi + converged || block 'installed deployer state is absent or drifted; repair before resume' + policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer health check failed; repair before resume' + health=healthy + ((was_drained == 1)) || { report NO_CHANGE no ready-to-deploy "$(rollback_available)"; return; } + rm -f "$drained" + sync -f "$state_root" 2>/dev/null || block 'cleared drain marker is not durable' + report CHANGED yes ready-to-deploy "$(rollback_available)" +} + +perform_uninstall() { + local changed=no unit managed_present=no + if [[ -e "$state_root" || -L "$state_root" || -e "$lock_root" || -L "$lock_root" || -e "$current" || -L "$current" ]]; then managed_present=yes; fi + for unit in "${unit_names[@]}"; do [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/$unit" ]] || managed_present=yes; done + if [[ $managed_present == no ]]; then + for unit in "${timer_names[@]}"; do + systemctl is-enabled "$unit" >/dev/null 2>&1 || systemctl is-active "$unit" >/dev/null 2>&1 || continue + managed_present=yes + done + fi + if [[ "$managed_present" == no ]]; then report NO_CHANGE no retained-state "$(rollback_available)"; return; fi + if active_deployment; then block 'active deployment prevents this operation'; fi + acquire_lock + secure_directory "$state_root" 700 1 + for unit in "${unit_names[@]}"; do [[ ! -e "$systemd_root/$unit" || -L "$systemd_root/$unit" || -f "$systemd_root/$unit" ]] || block "managed unit $unit has an unsafe type"; done + if [[ -e "$drained" || -L "$drained" ]]; then + secure_file "$drained" 'drain marker' + else + temporary=$(mktemp "$state_root/.drained.XXXXXX") + chmod 0600 "$temporary" + mv -Tf "$temporary" "$drained" + # The drain guard must be durable before the first mutation beneath it. + sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || block 'drain marker is not durable' + sync -f "$state_root" 2>/dev/null || block 'drain marker publication is not durable' + fi + if active_deployment; then block 'active deployment started while draining'; fi + for unit in "${timer_names[@]}"; do + if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]] || systemctl is-enabled "$unit" >/dev/null 2>&1 || systemctl is-active "$unit" >/dev/null 2>&1; then + systemctl disable --now "$unit" >/dev/null 2>&1 || block 'deployer timers did not stop during uninstall' + changed=yes + fi + done + if [[ -L "$current" ]]; then + [[ -d "$install_root" && ! -L "$install_root" && $(stat -c '%u:%a' "$install_root") == "$expected_uid:755" ]] || block 'managed install boundary has an unsafe owner, mode, or type' + rm -f "$current"; changed=yes + fi + [[ ! -e "$current" ]] || block 'activation pointer has an unsafe type' + for unit in "${unit_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then rm -f "$systemd_root/$unit"; changed=yes; fi; done + systemctl daemon-reload >/dev/null 2>&1 || block 'systemd manager reload failed after unit removal' + # Persist pointer, unit, and timer removals across their filesystems before + # the drain marker guarding this maintenance window is cleared. + if [[ -d "$install_root" && ! -L "$install_root" ]]; then + sync -f "$install_root" 2>/dev/null || block 'uninstalled install root is not durable' + fi + sync -f "$systemd_root" 2>/dev/null || block 'uninstalled systemd boundary is not durable' + sync -f "$state_root" 2>/dev/null || block 'uninstalled host state is not durable' + rm -f "$drained" "$active_operation" + sync -f "$state_root" 2>/dev/null || block 'cleared drain marker is not durable' + if [[ "$changed" == yes ]]; then report CHANGED yes retained-state "$(rollback_available)"; else report NO_CHANGE no retained-state "$(rollback_available)"; fi +} + +validate_config +if [[ "$mode" == drain || "$mode" == uninstall || "$mode" == rollback ]]; then + require_maintenance_host +else + validate_checkout + require_host +fi +case "$mode" in + check) acquire_check_lock; perform_check ;; + install|upgrade|repair) acquire_lock + if ((recovered_rollback)); then health=healthy; report CHANGED yes restore-host-policy-evidence-then-check no; else perform_converge; fi ;; + rollback) acquire_lock; perform_rollback ;; + drain) acquire_lock; perform_drain ;; + resume) acquire_lock; perform_resume ;; + uninstall) perform_uninstall ;; +esac diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index ad15656..3efb82d 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -1022,8 +1022,16 @@ perform_check() { note "CHECK_OK controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" } +reject_deployer_host() { + # A deployer host is a separate role; never run controller mutations on it. + if compgen -G "$(root_path /etc/systemd/system)/ci-fleet-deployer*" >/dev/null || [[ -e "$(root_path /var/lib/ci-fleet-deployer)" || -e "$(root_path /etc/ci-fleet-deployer)" ]]; then + die 'deployer host state is present; controller and deployer roles are separate hosts' + fi +} + perform_converge() { local count existing_status desired_controller_id=$controller_id + reject_deployer_host if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then die '--upgrade requires an existing managed installation; use --install or --adopt' fi @@ -1063,6 +1071,7 @@ latest_checkpoint() { } perform_rollback() { + reject_deployer_host checkpoint_dir=$(latest_checkpoint) [[ -n "$checkpoint_dir" ]] || die 'no controller checkpoint is available' load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" @@ -1106,6 +1115,12 @@ else exec 9>"$lock_file" flock -n 9 || die 'another ci-fleet installer or drift check is already running' fi +# Role admission is serialized across the controller and deployer installers: +# both flock this shared path for their whole mutating run. +role_lock=$(root_path /run/ci-fleet-role-admission.lock) +install -d -m 0755 "$(dirname "$role_lock")" +exec 8>"$role_lock" +flock -n 8 || die 'another ci-fleet role installation is already running' case "$mode" in check|install|adopt|upgrade) validate_common_arguments diff --git a/scripts/test-deployer-units.sh b/scripts/test-deployer-units.sh new file mode 100755 index 0000000..0ec0cfd --- /dev/null +++ b/scripts/test-deployer-units.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +command -v systemd-analyze >/dev/null || { printf 'systemd-analyze is required\n' >&2; exit 1; } +[[ -d /usr/lib/systemd/system ]] || { printf 'systemd unit library is missing\n' >&2; exit 1; } +root=$(mktemp -d) +trap 'rm -rf "$root"' EXIT +install -d -m 0755 "$root/etc/systemd/system" \ + "$root/opt/ci-fleet-deployer/current/scripts" "$root/usr/lib/systemd" +cp -a /usr/lib/systemd/system "$root/usr/lib/systemd/" +install -m 0755 "$repo_root/scripts/deployer-runtime.sh" \ + "$root/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh" +install -m 0644 "$repo_root"/deploy/deployer/* "$root/etc/systemd/system/" +systemd-analyze verify --root="$root" \ + ci-fleet-deployer.service \ + ci-fleet-deployer-health.service ci-fleet-deployer-health.timer \ + ci-fleet-deployer-cleanup.service ci-fleet-deployer-cleanup.timer \ + ci-fleet-deployer-drain.service +printf 'DEPLOYER_UNIT_TESTS_OK\n' diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh new file mode 100755 index 0000000..6165fa3 --- /dev/null +++ b/scripts/test-install-deployer.sh @@ -0,0 +1,1825 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +installer=$repo_root/scripts/install-deployer.sh +runtime=$repo_root/scripts/deployer-runtime.sh + +fail() { printf 'FAIL(line %s): %s\n' "${BASH_LINENO[0]:-?}" "$*" >&2; exit 1; } +expect_success() { + local output line=${BASH_LINENO[0]} + output=$("$@" 2>&1) || { printf 'FAIL(line %s): expected success: %s; output=%s\n' "$line" "$*" "$output" >&2; exit 1; } + printf '%s\n' "$output" +} +expect_failure() { + local expected=$1 output line=${BASH_LINENO[0]} + shift + if output=$("$@" 2>&1); then printf 'FAIL(line %s): expected failure: %s; output=%s\n' "$line" "$*" "$output" >&2; exit 1; fi + grep -Fq -- "$expected" <<<"$output" || fail "missing [$expected]: $output" + printf '%s\n' "$output" +} + +[[ -x "$installer" && -x "$runtime" ]] || fail 'deployer installer/runtime is missing' +expect_failure 'an explicit operating mode is required' "$installer" + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +root=$tmp/root +fake_bin=$tmp/bin +mkdir -p "$fake_bin" "$root/etc/ci-fleet-deployer/adapters" \ + "$root/etc/ci-fleet-deployer/credentials" "$root/etc/ci-fleet-deployer/evidence" \ + "$root/etc/systemd/system" "$root/run/systemd/system" +printf 'ID=debian\nVERSION_ID="12"\n' >"$root/etc/os-release" +chmod 0700 "$root/etc/ci-fleet-deployer" "$root/etc/ci-fleet-deployer/adapters" \ + "$root/etc/ci-fleet-deployer/credentials" "$root/etc/ci-fleet-deployer/evidence" + +cat >"$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +set -u +case "${1:-}" in + info) exit "${FAKE_DOCKER_INFO_EXIT:-0}" ;; + compose) [[ "${2:-}" == version ]] && exit "${FAKE_COMPOSE_EXIT:-0}" ;; + ps) [[ -z "${FAKE_DOCKER_PS:-}" ]] || printf '%s\n' "$FAKE_DOCKER_PS" ;; + network) [[ "${2:-}" == ls ]] && { [[ -z "${FAKE_DOCKER_NETWORKS:-}" ]] || printf '%s\n' "$FAKE_DOCKER_NETWORKS"; exit 0; } ;; + volume) [[ "${2:-}" == ls ]] && { [[ -z "${FAKE_DOCKER_VOLUMES:-}" ]] || printf '%s\n' "$FAKE_DOCKER_VOLUMES"; exit 0; } ;; +esac +exit 0 +EOF +cat >"$fake_bin/systemctl" <<'EOF' +#!/usr/bin/env bash +set -u +root=${CI_FLEET_DEPLOYER_ROOT:-} +log=${FAKE_SYSTEMCTL_LOG:-/dev/null} +printf '%s\n' "$*" >>"$log" +[[ -z ${FAKE_SYSTEMCTL_FAIL_COMMAND:-} || ${1:-} != "$FAKE_SYSTEMCTL_FAIL_COMMAND" ]] || exit 1 +case "${1:-}" in + is-system-running) + [[ -z "${FAKE_SYSTEMD_FAIL:-}" ]] || exit 1 + printf '%s\n' "${FAKE_SYSTEMD_STATE:-running}" + [[ ${FAKE_SYSTEMD_STATE:-running} != degraded ]] + ;; + is-enabled|is-active) + [[ -e "$root/var/lib/ci-fleet-deployer/unit-${2:-}" ]] || exit 1 + if [[ $1 == is-enabled ]]; then + if [[ -e "$root/var/lib/ci-fleet-deployer/unit-${2:-}.runtime" ]]; then printf 'enabled-runtime\n'; else printf '%s\n' "${FAKE_SYSTEMD_IS_ENABLED_OUTPUT:-enabled}"; fi + fi + ;; + enable) + shift + runtime_flag=0 + [[ "${1:-}" != --runtime ]] || { runtime_flag=1; shift; } + [[ "${1:-}" != --now ]] || shift + for unit in "$@"; do + if ((runtime_flag)); then : >"$root/var/lib/ci-fleet-deployer/unit-$unit.runtime"; else rm -f "$root/var/lib/ci-fleet-deployer/unit-$unit.runtime"; fi + : >"$root/var/lib/ci-fleet-deployer/unit-$unit" + done ;; + disable) + shift + [[ "${1:-}" != --now ]] || shift + for unit in "$@"; do rm -f "$root/var/lib/ci-fleet-deployer/unit-$unit" "$root/var/lib/ci-fleet-deployer/unit-$unit.runtime"; done ;; +esac +exit 0 +EOF +cat >"$fake_bin/systemd-analyze" <<'EOF' +#!/usr/bin/env bash +[[ "${1:-}" == verify ]] || exit 1 +exit "${FAKE_SYSTEMD_VERIFY_EXIT:-0}" +EOF +cat >"$fake_bin/timedatectl" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "${FAKE_TIME_SYNC:-yes}" +EOF +cat >"$fake_bin/curl" <<'EOF' +#!/usr/bin/env bash +cat >/dev/null +exit "${FAKE_CURL_EXIT:-0}" +EOF +cat >"$fake_bin/systemd-inhibit" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +while (($#)); do + [[ "$1" != -- ]] || { shift; exec "$@"; } + shift +done +exit 2 +EOF +cat >"$fake_bin/df" <<'EOF' +#!/usr/bin/env bash +if [[ -n ${FAKE_DF_REQUIRE_EXISTING:-} && ! -e ${!#} ]]; then exit 1; fi +printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\nfixture 104857600 1 %s 1%% /\n' "${FAKE_DISK_AVAILABLE:-104857599}" +EOF +chmod 0755 "$fake_bin"/* +export PATH="$fake_bin:$PATH" +export FAKE_SYSTEMCTL_LOG=$tmp/systemctl.log +export CI_FLEET_DEPLOYER_TESTING=1 CI_FLEET_DEPLOYER_ROOT=$root +export CI_FLEET_DEPLOYER_EUID_OVERRIDE=0 CI_FLEET_DEPLOYER_TEST_NETWORK=ok + +adapter=$root/etc/ci-fleet-deployer/adapters/application-adapter +cat >"$adapter" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +printf '%s\n' "$1" >>"${FAKE_ADAPTER_LOG:?}" +[[ -z ${FAKE_ADAPTER_RECORD_CONFIG:-} ]] || { printf '%s\n' "${CI_FLEET_DEPLOYER_CONFIG:-unset}" >"$FAKE_ADAPTER_RECORD_CONFIG"; : >"$FAKE_ADAPTER_RECORD_CONFIG.seen"; cp "${CI_FLEET_DEPLOYER_CONFIG:-/dev/null}" "$FAKE_ADAPTER_RECORD_CONFIG.content" 2>/dev/null || true; } +if [[ -n ${FAKE_ADAPTER_FORBID_CONFIG_PATH:-} && ${CI_FLEET_DEPLOYER_CONFIG:-} == "$FAKE_ADAPTER_FORBID_CONFIG_PATH" ]]; then exit 43; fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_FORBID_REQUEST_PATH:-} && ${CI_FLEET_DEPLOYER_REQUEST:-} == "$FAKE_ADAPTER_FORBID_REQUEST_PATH" ]]; then exit 44; fi +if [[ -n ${FAKE_ADAPTER_REPLACE_PATH:-} && -e $FAKE_ADAPTER_REPLACE_PATH ]]; then + mv "$FAKE_ADAPTER_REPLACE_PATH" "$FAKE_ADAPTER_REPLACE_PATH.saved" + printf '#!/usr/bin/env bash\nexit 45\n' >"$FAKE_ADAPTER_REPLACE_PATH" + chmod 0700 "$FAKE_ADAPTER_REPLACE_PATH" +fi +if [[ ${FAKE_ADAPTER_SLEEP_OPERATION:-} == "$1" ]]; then sleep 2; fi +if [[ "$1" == health && -n ${FAKE_ADAPTER_FAIL_HEALTH_AFTER:-} ]]; then + health_calls=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) + ((health_calls <= FAKE_ADAPTER_FAIL_HEALTH_AFTER)) || exit 42 +fi +if [[ -e "${FAKE_ADAPTER_FAIL:-/nonexistent}" ]]; then + fail_operation=$(<"$FAKE_ADAPTER_FAIL") + [[ "$fail_operation" != all && "$fail_operation" != "$1" ]] || exit 42 +fi +case "$1" in validate|health|cleanup|deploy|rollback) ;; *) exit 2 ;; esac +if [[ "$1" == rollback && -n ${CI_FLEET_DEPLOYER_ROLLBACK_COMMIT:-} ]]; then + install -m 0600 /dev/null "$CI_FLEET_DEPLOYER_ROLLBACK_COMMIT" +fi +if [[ "$1" == rollback && -n ${FAKE_ADAPTER_FAIL_AFTER_MARKER:-} ]]; then exit 42; fi +if [[ -n ${FAKE_ADAPTER_DELETE_CONSUMED_GLOB:-} ]]; then rm -rf $FAKE_ADAPTER_DELETE_CONSUMED_GLOB; fi +if [[ -n ${FAKE_ADAPTER_MUTATE_AUDIT_PATH:-} ]]; then + if [[ ${FAKE_ADAPTER_MUTATE_AUDIT_MODE:-symlink} == unlink ]]; then + rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH"; printf 'adapter-replacement\n' >"$FAKE_ADAPTER_MUTATE_AUDIT_PATH" + else + rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" + ln -s "$FAKE_ADAPTER_MUTATE_AUDIT_TARGET" "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" + fi +fi +if [[ -n ${FAKE_ADAPTER_CHMOD_DURING:-} ]]; then chmod 0644 "$FAKE_ADAPTER_CHMOD_DURING"; fi +if [[ -n ${FAKE_ADAPTER_CHOWN_DURING:-} && $EUID == 0 ]]; then chown 65534 "$FAKE_ADAPTER_CHOWN_DURING"; fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_INCUMBENT_PATH:-} ]]; then + chmod 0755 "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH" + chmod 0644 "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH"/policy.conf "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH"/state.json + if ((EUID == 0)); then chown 65534 "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH" "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH"/policy.conf "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH"/state.json; fi +fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_LKG_ROOT:-} ]]; then + chmod 0644 "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good.json "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good-policy.conf + if ((EUID == 0)); then chown 65534 "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good.json "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good-policy.conf; fi +fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_DELETE_STATE_ROOT:-} ]]; then rm -rf -- "$FAKE_ADAPTER_DELETE_STATE_ROOT"; fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT:-} ]]; then + snapshot=$(find "$FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT" -maxdepth 1 -type d -name '.snapshot.*' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-) + [[ -n "$snapshot" ]] && printf 'adapter-mutation\n' >>"$snapshot/policy.conf" +fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_SIGNAL_PPID:-} ]]; then kill -TERM "$PPID"; sleep 5; fi +if [[ -e "${FAKE_ADAPTER_FAIL_AFTER_MUTATION:-/nonexistent}" ]]; then exit 42; fi +EOF +chmod 0700 "$adapter" +export FAKE_ADAPTER_LOG=$tmp/adapter.log +credential=$root/etc/ci-fleet-deployer/credentials/application.credential +printf 'CANARY_SECRET_VALUE_DO_NOT_PRINT\n' >"$credential" +chmod 0600 "$credential" +approval=$root/etc/ci-fleet-deployer/evidence/approval.conf +checkpoint=$root/etc/ci-fleet-deployer/evidence/checkpoint.conf +production_gate=$root/etc/ci-fleet-deployer/evidence/production-authorization.conf +core_ref=$(git -C "$repo_root" rev-parse HEAD) +source_ref=1111111111111111111111111111111111111111 +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence() { + local evidence_environment=${1:-staging} evidence_target=${2:-example-staging} + cat >"$approval" <"$checkpoint" <"$production_gate" <"$root/etc/ci-fleet-deployer/deployer.conf" <>"$root/etc/ci-fleet-deployer/deployer.conf"; fi + chmod 0600 "$root/etc/ci-fleet-deployer/deployer.conf" +} +# snapshot_state : record the full current managed state into a +# recovery-transaction fixture so restoration returns to a valid installation. +snapshot_state() { + local tx=$1 name + for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$tx/state/$name"; printf '%s\n' "$name" >>"$tx/state-present"; } + done + printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$tx/current-target" +} +write_evidence +write_config +config=$root/etc/ci-fleet-deployer/deployer.conf + +FAKE_DF_REQUIRE_EXISTING=1 expect_failure 'installed deployer lock boundary is absent or unsafe' "$installer" --check --config "$config" >/dev/null + +write_evidence production example-production +write_config production example-production +expect_failure 'production authorization evidence must be a regular file' "$installer" --check --config "$config" >/dev/null +write_production_gate +expect_failure 'installed deployer lock boundary is absent or unsafe' "$installer" --check --config "$config" >/dev/null +rm "$production_gate" +write_evidence +write_config + +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-99-99T99:99:99Z')) +PY +expect_failure 'approval timestamp must be UTC RFC3339' "$installer" --check --config "$config" >/dev/null +write_evidence +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ENVIRONMENT=staging', 'ENVIRONMENT=bad result=CHANGED')) +PY +unsafe_report=$(expect_failure 'invalid explicit environment' "$installer" --check --config "$config") +grep -Fq 'environment=unknown target=unknown' <<<"$unsafe_report" || fail 'report exposed unvalidated configuration fields' +[[ "$unsafe_report" != *'environment=bad result=CHANGED'* ]] || fail 'report allowed field injection' +write_config + +# Read-only preflight and strict policy variants fail closed without reading secrets. +cp "$root/etc/os-release" "$tmp/os-release" +printf 'ID=alpine\nVERSION_ID=3.20\n' >"$root/etc/os-release" +expect_failure 'unsupported Linux distribution or release' "$installer" --check --config "$config" >/dev/null +cp "$tmp/os-release" "$root/etc/os-release" +FAKE_TIME_SYNC=no expect_failure 'host time is not synchronized' "$installer" --check --config "$config" >/dev/null +FAKE_SYSTEMD_FAIL=1 expect_failure 'systemd is unavailable' "$installer" --check --config "$config" >/dev/null +FAKE_DOCKER_INFO_EXIT=1 expect_failure 'Docker Engine is unavailable' "$installer" --check --config "$config" >/dev/null +CI_FLEET_DEPLOYER_TEST_NETWORK=fail expect_failure 'network prerequisite DNS lookup failed' "$installer" --check --config "$config" >/dev/null +FAKE_CURL_EXIT=1 expect_failure 'network prerequisite HTTPS check failed' "$installer" --check --config "$config" >/dev/null +FAKE_DISK_AVAILABLE=1 expect_failure 'insufficient deployer disk capacity' "$installer" --check --config "$config" >/dev/null +FAKE_COMPOSE_EXIT=1 expect_failure 'Docker Compose v2 is required but unavailable' "$installer" --check --config "$config" >/dev/null + +chmod 0755 "$root/etc/ci-fleet-deployer" +expect_failure 'unsafe managed directory' "$installer" --check --config "$config" >/dev/null +chmod 0700 "$root/etc/ci-fleet-deployer" +mv "$root/etc/ci-fleet-deployer/adapters" "$root/etc/ci-fleet-deployer/adapters.real" +ln -s adapters.real "$root/etc/ci-fleet-deployer/adapters" +expect_failure 'unsafe symlinked managed directory' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/ci-fleet-deployer/adapters" +mv "$root/etc/ci-fleet-deployer/adapters.real" "$root/etc/ci-fleet-deployer/adapters" + +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ARTIFACT_IMAGE=registry.example.invalid/example/app@sha256:', 'ARTIFACT_IMAGE=registry.example.invalid/example/app:latest#')) +PY +expect_failure 'artifact image must be an immutable' "$installer" --check --config "$config" >/dev/null +write_config + +capability=$root/etc/ci-fleet-deployer/evidence/github-capability.conf +python3 - "$config" "$capability" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=manual-exact-head', 'APPROVAL_PROVIDER=github-environment') + 'APPROVAL_CAPABILITY_EVIDENCE_PATH='+sys.argv[2]+'\n') +PY +expect_failure 'GitHub capability evidence must be a regular file' "$installer" --check --config "$config" >/dev/null +cat >"$capability" </dev/null +python3 - "$capability" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text('\n'.join(x for x in p.read_text().splitlines() if not x.startswith('CAPABILITY_ID='))+'\n') +PY +expect_failure 'GitHub Environment capability evidence is missing identity or UTC time' "$installer" --check --config "$config" >/dev/null +cat >"$capability" </dev/null +install -m 0600 "$tmp/capability.saved" "$capability" +write_config +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external').replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=external:example-vault:staging/deployer')) +PY +expect_failure 'installed deployer lock boundary is absent or unsafe' "$installer" --check --config "$config" >/dev/null +write_config + +printf 'UNKNOWN=value\n' >"$tmp/unknown.conf"; chmod 0600 "$tmp/unknown.conf" +expect_failure 'configuration path must be inside' "$installer" --check --config "$tmp/unknown.conf" >/dev/null +printf 'UNKNOWN=value\n' >"$config" +expect_failure 'unknown configuration key: UNKNOWN' "$installer" --check --config "$config" >/dev/null +write_config +printf 'SCHEMA_VERSION=1\n' >>"$config" +expect_failure 'duplicate configuration key: SCHEMA_VERSION' "$installer" --check --config "$config" >/dev/null +write_config + +export CI_FLEET_DEPLOYER_EUID_OVERRIDE=1000 +expect_failure 'run this mode as root' "$installer" --install --config "$config" >/dev/null +export CI_FLEET_DEPLOYER_EUID_OVERRIDE=0 + +before=$(find "$root" -printf '%P %y %m\n' | sort | sha256sum) +expect_failure 'result=BLOCKED' "$installer" --check --config "$config" >/dev/null +after=$(find "$root" -printf '%P %y %m\n' | sort | sha256sum) +[[ "$before" == "$after" ]] || fail '--check changed the test host' +fresh_uninstall=$(expect_success "$installer" --uninstall --config "$config") +[[ "$after" == "$(find "$root" -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'fresh uninstall mutated an unmanaged host' +grep -Fq 'result=NO_CHANGE' <<<"$fresh_uninstall" || fail 'fresh uninstall did not report NO_CHANGE' + +mkdir -p "$root/usr/lib" +cp "$root/etc/os-release" "$root/usr/lib/os-release" +rm "$root/etc/os-release" +ln -s ../usr/lib/os-release "$root/etc/os-release" +export FAKE_ADAPTER_FORBID_CONFIG_PATH=$config +first=$(expect_success "$installer" --install --config "$config") +unset FAKE_ADAPTER_FORBID_CONFIG_PATH +rm "$root/etc/os-release" "$root/usr/lib/os-release" +cp "$tmp/os-release" "$root/etc/os-release" +grep -Fq 'REPORT action=install result=CHANGED environment=staging' <<<"$first" || fail 'fresh install report is incomplete' +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'fresh install lacks atomic current release' +[[ $(stat -c %a "$root/var/lib/ci-fleet-deployer/install-state.json") == 600 ]] || fail 'install state mode is not 0600' +for unit in ci-fleet-deployer.service ci-fleet-deployer-health.service ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.service ci-fleet-deployer-cleanup.timer ci-fleet-deployer-drain.service; do + [[ -f "$root/etc/systemd/system/$unit" ]] || fail "missing unit $unit" +done +second=$(expect_success "$installer" --install --config "$config") +grep -Fq 'result=NO_CHANGE' <<<"$second" || fail 'second install was not idempotent' +check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'REPORT action=check result=NO_CHANGE' <<<"$check" || fail 'check did not report convergence' +FAKE_SYSTEMD_STATE=degraded expect_success "$installer" --check --config "$config" >/dev/null +chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +expect_failure 'managed systemd unit has an unsafe owner, mode, or type' "$installer" --repair --config "$config" >/dev/null +chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ $(stat -c %a "$root/etc/systemd/system/ci-fleet-deployer.service") == 644 ]] || fail 'repair did not restore unit mode 0644' + +# Repair must replace a damaged active release from the validated checkout. +printf 'truncated\n' >"$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +cmp -s "$repo_root/scripts/deployer-runtime.sh" "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh" || fail 'repair did not replace the damaged release with reviewed bytes' +expect_success "$installer" --check --config "$config" >/dev/null + +preparing=$root/var/lib/ci-fleet-deployer/.transaction-preparing.interrupted +mkdir -m 0700 "$preparing" +state_before_preparing=$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json") +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$preparing" && "$state_before_preparing" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'incomplete transaction preparation was treated as recovery state' + +interrupted=$root/var/lib/ci-fleet-deployer/.transaction.interrupted +mkdir -m 0700 "$interrupted" "$interrupted/units" "$interrupted/state" +for name in install-state.json active-policy.conf; do + cp "$root/var/lib/ci-fleet-deployer/$name" "$interrupted/state/$name" + printf '%s\n' "$name" >>"$interrupted/state-present" +done +for path in "$root"/etc/systemd/system/ci-fleet-deployer*; do + name=${path##*/}; cp "$path" "$interrupted/units/$name"; printf '%s\n' "$name" >>"$interrupted/units-present" +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$interrupted/current-target" +printf '%s\n' ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer >"$interrupted/timers-enabled" +printf 'interrupted\n' >>"$root/var/lib/ci-fleet-deployer/install-state.json" +printf 'interrupted\n' >>"$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'interrupted installer transaction requires recovery' "$installer" --check --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$interrupted" ]] || fail 'interrupted transaction was not recovered' +expect_success "$installer" --check --config "$config" >/dev/null + +# An unsafe transaction manifest must fail closed before any recovery mutation. +unsafe_tx=$root/var/lib/ci-fleet-deployer/.transaction.unsafe +mkdir -m 0700 "$unsafe_tx" "$unsafe_tx/units" "$unsafe_tx/state" +cp "$root/var/lib/ci-fleet-deployer/install-state.json" "$unsafe_tx/state/install-state.json" +printf 'install-state.json\n' >"$unsafe_tx/state-present" +printf 'rogue-unit.service\n' >"$unsafe_tx/units-present" +units_before_unsafe=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum) +expect_failure 'transaction unit manifest is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ "$units_before_unsafe" == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum)" ]] || fail 'unsafe transaction manifest mutated installed units' +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'unsafe transaction manifest removed the activation pointer' +rm "$unsafe_tx/units-present" +printf 'not-a-release\n' >"$unsafe_tx/current-target" +expect_failure 'transaction current pointer is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'unsafe transaction pointer removed the activation pointer' +rm "$unsafe_tx/current-target" +mkdir "$unsafe_tx/timers-enabled" +expect_failure 'transaction manifest timers-enabled has an unsafe type' "$installer" --repair --config "$config" >/dev/null +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'unsafe manifest type removed the activation pointer' +rmdir "$unsafe_tx/timers-enabled" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$unsafe_tx" ]] || fail 'corrected transaction was not recovered' +expect_success "$installer" --check --config "$config" >/dev/null + +# A truncated or drifted transaction manifest must block before any recovery mutation. +truncated_tx=$root/var/lib/ci-fleet-deployer/.transaction.truncated +mkdir -m 0700 "$truncated_tx" "$truncated_tx/units" "$truncated_tx/state" +for name in install-state.json active-policy.conf; do + cp "$root/var/lib/ci-fleet-deployer/$name" "$truncated_tx/state/$name" + printf '%s\n' "$name" >>"$truncated_tx/state-present" +done +sed -i '$d' "$truncated_tx/state-present" +state_before_truncated=$(find "$root/var/lib/ci-fleet-deployer" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum) +expect_failure 'transaction state manifest does not match its backup directory' "$installer" --repair --config "$config" >/dev/null +[[ "$state_before_truncated" == "$(find "$root/var/lib/ci-fleet-deployer" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum)" ]] || fail 'truncated transaction manifest mutated managed state' +printf 'active-policy.conf\n' >>"$truncated_tx/state-present" +cp "$root/etc/systemd/system/ci-fleet-deployer.service" "$truncated_tx/units/ci-fleet-deployer.service" +expect_failure 'transaction units manifest is missing but backups remain' "$installer" --repair --config "$config" >/dev/null +rm -rf -- "$truncated_tx" +expect_success "$installer" --check --config "$config" >/dev/null + +# Transaction recovery must restore runtime-only timer enablement exactly. +runtime_tx=$root/var/lib/ci-fleet-deployer/.transaction.runtime-enabled +mkdir -m 0700 "$runtime_tx" "$runtime_tx/units" "$runtime_tx/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$runtime_tx/state/$name"; printf '%s\n' "$name" >>"$runtime_tx/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$runtime_tx/current-target" +: >"$root/var/lib/ci-fleet-deployer/unit-ci-fleet-deployer-health.timer.runtime" +printf 'ci-fleet-deployer-health.timer\n' >"$runtime_tx/timers-enabled-runtime" +expect_success "$installer" --repair --config "$config" >/dev/null +grep -Fq 'enable --runtime ci-fleet-deployer-health.timer' "$FAKE_SYSTEMCTL_LOG" || fail 'recovery did not restore runtime-only enablement' +rm -f "$root/var/lib/ci-fleet-deployer/unit-ci-fleet-deployer-health.timer.runtime" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# Recovery must remove a snapshot published after the transaction began. +created_tx=$root/var/lib/ci-fleet-deployer/.transaction.created-snapshot +mkdir -m 0700 "$created_tx" "$created_tx/units" "$created_tx/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$created_tx/state/$name"; printf '%s\n' "$name" >>"$created_tx/state-present"; } +done +for path in "$root"/etc/systemd/system/ci-fleet-deployer*; do + name=${path##*/}; cp "$path" "$created_tx/units/$name"; printf '%s\n' "$name" >>"$created_tx/units-present" +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$created_tx/current-target" +printf '%s\n' ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer >"$created_tx/timers-enabled" +orphan_snapshot=$root/var/lib/ci-fleet-deployer/deployed/.snapshot.orphaned +mkdir -m 0700 "$orphan_snapshot" +install -m 0600 "$root/var/lib/ci-fleet-deployer/active-policy.conf" "$orphan_snapshot/policy.conf" +install -m 0600 "$root/var/lib/ci-fleet-deployer/install-state.json" "$orphan_snapshot/state.json" +current_deployed=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") +printf '%s\n' "$current_deployed" >"$created_tx/deployed-target" +printf '.snapshot.orphaned\n' >"$created_tx/deployed-created" +rm "$root/var/lib/ci-fleet-deployer/deployed/current" +ln -s .snapshot.orphaned "$root/var/lib/ci-fleet-deployer/deployed/current" +deployed_count_before=$(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$orphan_snapshot" ]] || fail 'recovery retained a snapshot created after the transaction began' +[[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == $((deployed_count_before - 1)) ]] || fail 'recovery removed the wrong snapshot' +[[ $(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") == "$current_deployed" ]] || fail 'recovery did not restore the prior deployed pointer' +expect_success "$installer" --check --config "$config" >/dev/null + +# Transaction recovery must tolerate timers whose unit files were never installed. +absent_timer_tx=$root/var/lib/ci-fleet-deployer/.transaction.absent-timer +mkdir -m 0700 "$absent_timer_tx" "$absent_timer_tx/units" "$absent_timer_tx/state" +snapshot_state "$absent_timer_tx" +mv "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" "$tmp/cleanup.timer.saved" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$absent_timer_tx" ]] || fail 'absent-timer transaction was not recovered' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# Read-only checks must not create checkout snapshots inside managed state. +if compgen -G "$root/var/lib/ci-fleet-deployer/.checkout.*" >/dev/null; then fail 'read-only check left a checkout snapshot in managed state'; fi + +# A noncanonical activation pointer must fail convergence instead of certifying it. +rm "$root/opt/ci-fleet-deployer/current" +ln -s "$root/opt/ci-fleet-deployer/releases/$core_ref" "$root/opt/ci-fleet-deployer/current" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +rm "$root/opt/ci-fleet-deployer/current" +ln -s "releases/$core_ref" "$root/opt/ci-fleet-deployer/current" +expect_success "$installer" --check --config "$config" >/dev/null + +rm "$root/etc/systemd/system/ci-fleet-deployer-health.timer" +expect_failure 'result=BLOCKED' "$installer" --check --config "$config" >/dev/null +repair=$(expect_success "$installer" --repair --config "$config") +grep -Fq 'result=CHANGED' <<<"$repair" || fail 'repair did not report a change' +[[ -f "$root/etc/systemd/system/ci-fleet-deployer-health.timer" ]] || fail 'repair did not restore unit drift' + +transaction_state=$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json") +printf '# transaction-checkpoint\n' >>"$root/etc/systemd/system/ci-fleet-deployer.service" +transaction_unit=$(sha256sum "$root/etc/systemd/system/ci-fleet-deployer.service") +FAKE_SYSTEMD_VERIFY_EXIT=1 expect_failure 'systemd unit verification failed' "$installer" --repair --config "$config" >/dev/null +[[ "$transaction_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'failed transaction replaced healthy state' +[[ "$transaction_unit" == "$(sha256sum "$root/etc/systemd/system/ci-fleet-deployer.service")" ]] || fail 'failed transaction did not restore units' +if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'failed transaction left recovery residue'; fi + +stale_stage="$root/opt/ci-fleet-deployer/releases/.${core_ref}.staging.interrupted" +mkdir "$stale_stage"; chmod 0755 "$stale_stage" +printf 'pid=999999\nstarted_at=1\n' >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$stale_stage" && ! -e "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'bounded stale transaction recovery did not converge' + +# A symlinked releases boundary must block before any staging cleanup deletion. +mv "$root/opt/ci-fleet-deployer/releases" "$tmp/releases.real" +ln -s "$tmp/releases.real" "$root/opt/ci-fleet-deployer/releases" +mkdir -m 0755 "$tmp/releases.real/.${core_ref}.staging.decoy" +expect_failure 'unsafe symlinked managed directory' "$installer" --repair --config "$config" >/dev/null +[[ -d "$tmp/releases.real/.${core_ref}.staging.decoy" ]] || fail 'symlinked releases boundary allowed staging deletion' +rm "$root/opt/ci-fleet-deployer/releases" +mv "$tmp/releases.real" "$root/opt/ci-fleet-deployer/releases" +rmdir "$root/opt/ci-fleet-deployer/releases/.${core_ref}.staging.decoy" +expect_success "$installer" --check --config "$config" >/dev/null + +# A runtime-only timer enablement must not satisfy convergence. +expect_success "$installer" --check --config "$config" >/dev/null +export FAKE_SYSTEMD_IS_ENABLED_OUTPUT=enabled-runtime +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +unset FAKE_SYSTEMD_IS_ENABLED_OUTPUT +expect_success "$installer" --check --config "$config" >/dev/null + +# A missing deployed rollback snapshot must break convergence before resume. +deployed_target=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") +rm "$root/var/lib/ci-fleet-deployer/deployed/current" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +ln -s "$deployed_target" "$root/var/lib/ci-fleet-deployer/deployed/current" +expect_success "$installer" --check --config "$config" >/dev/null +printf 'pid=%s\nstarted_at=1\n' "$$" >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'old operation marker survived PID reuse' +printf 'pid=%s\nstarted_at=1\n' "$$" >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +CI_FLEET_DEPLOYER_TEST_LIVE_PID=$$ expect_failure 'active deployment prevents this operation' "$installer" --repair --config "$config" >/dev/null +CI_FLEET_DEPLOYER_TEST_LIVE_PID=$$ expect_failure 'active deployment prevents this operation' "$installer" --upgrade --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'live stale-aged operation marker was expired' +rm "$root/var/lib/ci-fleet-deployer/active-operation" + +# An unsafe active-operation marker type must fail checks, not certify them. +mkdir "$root/var/lib/ci-fleet-deployer/active-operation" +expect_failure 'active operation marker has an unsafe type' "$installer" --check --config "$config" >/dev/null +expect_failure 'active operation marker has an unsafe type' "$installer" --repair --config "$config" >/dev/null +rmdir "$root/var/lib/ci-fleet-deployer/active-operation" +expect_success "$installer" --check --config "$config" >/dev/null + +# A stale marker for a dead process must fail read-only checks until a mutating recovery removes it. +printf 'pid=999999\nstarted_at=1\n' >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +expect_failure 'stale operation marker requires a mutating recovery' "$installer" --check --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'repair retained the stale operation marker' +expect_success "$installer" --check --config "$config" >/dev/null + +exec 8<"$root/var/lock/ci-fleet-deployer" +flock -n 8 || fail 'fixture could not acquire installer lock' +expect_failure 'another deployer installer operation is running' "$installer" --repair --config "$config" >/dev/null +expect_failure 'another deployer operation is running' "$installer" --check --config "$config" >/dev/null +flock -u 8; exec 8>&- + +mv "$root/var/lock/ci-fleet-deployer" "$root/var/lock/ci-fleet-deployer.real" +ln -s "$root/var/lock/ci-fleet-deployer.real" "$root/var/lock/ci-fleet-deployer" +expect_failure 'unsafe symlinked managed directory' "$installer" --repair --config "$config" >/dev/null +rm "$root/var/lock/ci-fleet-deployer" +mv "$root/var/lock/ci-fleet-deployer.real" "$root/var/lock/ci-fleet-deployer" + +mkdir -p "$root/run" +mv "$root/var/lock" "$root/run/lock" +ln -s ../run/lock "$root/var/lock" +expect_success "$installer" --check --config "$config" >/dev/null +rm "$root/var/lock"; mv "$root/run/lock" "$root/var/lock" + +unit_path=$root/etc/systemd/system/ci-fleet-deployer.service +mv "$unit_path" "$unit_path.real" +printf 'unrelated-unit\n' >"$tmp/unrelated-unit" +ln -s "$tmp/unrelated-unit" "$unit_path" +expect_failure 'managed systemd unit has an unsafe owner, mode, or type' "$installer" --repair --config "$config" >/dev/null +[[ $(<"$tmp/unrelated-unit") == unrelated-unit ]] || fail 'systemd unit symlink attack changed an unrelated file' +rm "$unit_path"; mv "$unit_path.real" "$unit_path" + +printf 'mixed-role\n' >"$root/etc/systemd/system/ci-fleet-health.service" +expect_failure 'ordinary CI controller or runner state is present' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/systemd/system/ci-fleet-health.service" +printf 'mixed-role\n' >"$root/etc/systemd/system/ci-fleet-drift.timer" +expect_failure 'ordinary CI controller or runner state is present' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/systemd/system/ci-fleet-drift.timer" +printf 'runner\n' >"$root/etc/systemd/system/actions.runner.example-org-example-repo.example-runner.service" +expect_failure 'ordinary GitHub Actions runner service is present' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/systemd/system/actions.runner.example-org-example-repo.example-runner.service" +export FAKE_DOCKER_PS='unrelated workload' +expect_failure 'unrelated Docker workload is present' "$installer" --check --config "$config" >/dev/null +unset FAKE_DOCKER_PS +export FAKE_DOCKER_PS='owned|deployer|staging-deployer-01' +expect_success "$installer" --check --config "$config" >/dev/null +export FAKE_DOCKER_PS='wrong|deployer|other-identity' +expect_failure 'unrelated Docker workload is present' "$installer" --check --config "$config" >/dev/null +unset FAKE_DOCKER_PS + +chmod 0644 "$credential" +secret_error=$(expect_failure 'credential file must be owner-only mode 0600' "$installer" --check --config "$config") +[[ "$secret_error" != *CANARY_SECRET_VALUE_DO_NOT_PRINT* ]] || fail 'credential content leaked in error output' +chmod 0600 "$credential" + +ln -s "$credential" "$root/etc/ci-fleet-deployer/credentials/symlinked" +python3 - "$config" "$root/etc/ci-fleet-deployer/credentials/symlinked" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_REF='+str(p.parent/'credentials/application.credential'), 'CREDENTIAL_REF='+sys.argv[2])) +PY +expect_failure 'credential reference must be a regular file, not a symlink' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/ci-fleet-deployer/credentials/symlinked" +write_config +python3 - "$config" "$root/etc/ci-fleet-deployer/credentials/./application.credential" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_REF='+str(p.parent/'credentials/application.credential'), 'CREDENTIAL_REF='+sys.argv[2])) +PY +expect_failure 'credential reference contains a symlink or non-canonical component' "$installer" --check --config "$config" >/dev/null +write_config + +printf 'bad\n' >>"$approval" +approval_error=$(expect_failure 'malformed approval evidence line' "$installer" --check --config "$config") +[[ "$approval_error" != *CANARY_SECRET_VALUE_DO_NOT_PRINT* ]] || fail 'secret content leaked beside evidence failure' +write_evidence + +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_IDENTITY=example-reviewer', 'APPROVAL_IDENTITY=alternate-reviewer')) +PY +expect_success "$installer" --repair --config "$config" >/dev/null +python3 - "$root/var/lib/ci-fleet-deployer/install-state.json" <<'PY' || fail 'approval identity drift was not recorded' +import json, sys +assert json.load(open(sys.argv[1]))['approval_identity'] == 'alternate-reviewer' +PY +write_evidence +expect_success "$installer" --repair --config "$config" >/dev/null + +mv "$root/var/lib/ci-fleet-deployer/active-policy.conf" "$root/var/lib/ci-fleet-deployer/active-policy.missing" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/active-policy.conf" ]] || fail 'repair did not restore a missing active policy' +rm "$root/var/lib/ci-fleet-deployer/active-policy.missing" + +printf '# force-transaction\n' >>"$root/etc/systemd/system/ci-fleet-deployer.service" +health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) +# Repair bypasses only the untrusted old-policy health probe, so the candidate +# health check after activation is the next (and only) adapter health call. +export FAKE_ADAPTER_FAIL_HEALTH_AFTER=$((health_calls_before)) FAKE_SYSTEMCTL_FAIL_COMMAND=disable +expect_failure 'candidate health check failed after activation' "$installer" --repair --config "$config" >/dev/null +unset FAKE_ADAPTER_FAIL_HEALTH_AFTER FAKE_SYSTEMCTL_FAIL_COMMAND +[[ $(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) == $((health_calls_before + 1)) ]] || fail 'repair did not run exactly the candidate health check' +compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null || fail 'failed restoration deleted its recovery transaction' +expect_success "$installer" --repair --config "$config" >/dev/null +if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'retry did not recover the retained transaction'; fi + +old_state=$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json") +new_image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +image=$new_image +write_evidence +write_config +deployed_current=$root/var/lib/ci-fleet-deployer/deployed/current +deployed_target=$(readlink "$deployed_current") +rm "$deployed_current" +expect_failure 'deployed rollback snapshot is missing; restore it before convergence' "$installer" --upgrade --config "$config" >/dev/null +ln -s "$deployed_target" "$deployed_current" +export FAKE_ADAPTER_SLEEP_OPERATION=validate CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS=1 +expect_failure 'candidate adapter validation failed' "$installer" --upgrade --config "$config" >/dev/null +unset FAKE_ADAPTER_SLEEP_OPERATION CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS +[[ "$old_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'timed-out candidate replaced healthy state' +export FAKE_ADAPTER_FAIL=$tmp/fail-adapter +printf 'validate\n' >"$FAKE_ADAPTER_FAIL" +expect_failure 'candidate adapter validation failed' "$installer" --upgrade --config "$config" >/dev/null +[[ "$old_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'failed candidate replaced healthy state' +rm "$FAKE_ADAPTER_FAIL"; unset FAKE_ADAPTER_FAIL + +chmod 0644 "$credential" +expect_failure 'credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null +chmod 0600 "$credential" + +# Candidate validate must revalidate credentials drifted after configuration validation. +export FAKE_ADAPTER_CHMOD_DURING=$credential +expect_failure 'candidate policy credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null +unset FAKE_ADAPTER_CHMOD_DURING +chmod 0600 "$credential" +[[ "$old_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'credential-drifted candidate replaced healthy state' + +install -m 0600 /dev/null "$root/var/lib/ci-fleet-deployer/drained" +health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) +export FAKE_ADAPTER_FAIL_HEALTH_AFTER=$((health_calls_before + 1)) +expect_failure 'candidate health check failed after activation' "$installer" --upgrade --config "$config" >/dev/null +unset FAKE_ADAPTER_FAIL_HEALTH_AFTER +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'failed convergence removed the drain marker' +rm "$root/var/lib/ci-fleet-deployer/drained" +upgrade=$(expect_success "$installer" --upgrade --config "$config") +grep -Fq 'result=CHANGED' <<<"$upgrade" || fail 'upgrade did not activate new immutable artifact' +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=approval-20260808-2')) +PY +expect_success "$installer" --upgrade --config "$config" >/dev/null +grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/last-known-good.json" || fail 'second undeployed upgrade replaced the deployed rollback point' + +# Failed active-policy health checks must not accumulate policy-check snapshots. +compgen -G "$root/var/lib/ci-fleet-deployer/.policy-check.*" >/dev/null && fail 'policy-check snapshot leaked before drifted-health regression' +export FAKE_ADAPTER_FAIL=$tmp/fail-adapter-health +printf 'health\n' >"$FAKE_ADAPTER_FAIL" +expect_failure 'active deployer health check failed' "$installer" --check --config "$config" >/dev/null +expect_failure 'active deployer health check failed' "$installer" --check --config "$config" >/dev/null +unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-adapter-health" +compgen -G "$root/var/lib/ci-fleet-deployer/.policy-check.*" >/dev/null && fail 'failed health validation leaked policy-check snapshots' +expect_success "$installer" --check --config "$config" >/dev/null + +# Deployed policy drift in an operational field must block promotion. +deployed_policy=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current")/policy.conf +python3 - "$deployed_policy" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external')) +PY +expect_failure 'deployed rollback policy has an invalid external secret-manager adapter reference' "$installer" --upgrade --config "$config" >/dev/null +python3 - "$deployed_policy" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=external', 'CREDENTIAL_PROVIDER=file')) +PY +expect_success "$installer" --check --config "$config" >/dev/null + +printf 'rollback\n' >"$tmp/fail-rollback" +# A candidate never deployed must not be a rollback target. +expect_failure 'no completed deployment is available to roll back to' "$installer" --rollback --config "$config" >/dev/null +# Rollback requires a completed deployment; simulate one for the rollback battery. +install -m 0600 /dev/null "$root/var/lib/ci-fleet-deployer/last-request.conf" +export FAKE_ADAPTER_FAIL=$tmp/fail-rollback +expect_failure 'application adapter rollback failed' "$installer" --rollback --config "$config" >/dev/null +unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-rollback" +grep -Fq 'sha256:bbbbbbbb' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'failed application rollback did not restore current core state' + +# An adapter that commits the rollback marker but exits nonzero must not be +# reported as an unchanged failure: recovery finalizes the committed rollback. +marker_rollback=$(FAKE_ADAPTER_FAIL_AFTER_MARKER=1 "$installer" --rollback --config "$config" 2>&1) || true +printf '%s\n' "$marker_rollback" | grep -Fq 'result=CHANGED' || fail 'marker-committed rollback was reported as an unchanged failure' +printf '%s\n' "$marker_rollback" | grep -Fq 'next=restore-host-policy-evidence-then-check' || fail 'marker-committed rollback lacks the operator reconciliation action' +grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'marker-committed rollback did not restore last-known-good state' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +# Rebuild the retained pair consumed by the marker-committed rollback. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' +rollback_calls_before=$(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) +FAKE_SYSTEMD_VERIFY_EXIT=1 expect_failure 'systemd unit verification failed' "$installer" --rollback --config "$config" >/dev/null +[[ $(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) == "$rollback_calls_before" ]] || fail 'application rollback ran before core rollback staging was proven' +grep -Fq 'sha256:bbbbbbbb' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'failed rollback did not preserve current core state' +cp "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$tmp/last-known-good.saved" +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good.json" <<'PY' +import json, sys +p=sys.argv[1]; value=json.load(open(p)); value['core_ref']='2222222222222222222222222222222222222222'; json.dump(value,open(p,'w'),indent=2,sort_keys=True) +PY +expect_failure 'last-known-good state and policy do not match' "$installer" --rollback --config "$config" >/dev/null +cp "$tmp/last-known-good.saved" "$root/var/lib/ci-fleet-deployer/last-known-good.json"; chmod 0600 "$root/var/lib/ci-fleet-deployer/last-known-good.json" +printf 'runner\n' >"$root/etc/systemd/system/actions.runner.rollback-drift.service" +expect_failure 'ordinary GitHub Actions runner service is present' "$installer" --rollback --config "$config" >/dev/null +[[ $(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) == "$rollback_calls_before" ]] || fail 'rollback adapter ran after role isolation drift' +rm "$root/etc/systemd/system/actions.runner.rollback-drift.service" +mv "$approval" "$approval.rollback-saved" +mv "$checkpoint" "$checkpoint.rollback-saved" +export FAKE_CURL_EXIT=1 +rollback=$(expect_success "$installer" --rollback --config "$config") +unset FAKE_CURL_EXIT +mv "$approval.rollback-saved" "$approval" +mv "$checkpoint.rollback-saved" "$checkpoint" +grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'rollback did not restore last-known-good state' +grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'rollback state lacks prior artifact' +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$rollback" || fail 'rollback report lacks the exact operator reconciliation action' +grep -Fq 'sha256:bbbbbbbb' "$config" || fail 'rollback unexpectedly rewrote operator-owned desired policy' + +# An interrupted committed rollback must report recovery without converging. +recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.rollback-interrupted +rm -rf "$root/var/lib/ci-fleet-deployer/deployed" +mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$recovery_transaction/state/$name"; printf '%s\n' "$name" >>"$recovery_transaction/state-present"; } +done +for path in "$root"/etc/systemd/system/ci-fleet-deployer*; do + name=${path##*/}; cp "$path" "$recovery_transaction/units/$name"; printf '%s\n' "$name" >>"$recovery_transaction/units-present" +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$recovery_transaction/current-target" +printf '%s\n' ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer >"$recovery_transaction/timers-enabled" +install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) +recovery=$(expect_success "$installer" --upgrade --config "$config") +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$recovery" || fail 'interrupted committed rollback recovery lacks the operator reconciliation action' +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'convergence ran after committed rollback recovery' +[[ $(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) == "$health_calls_before" ]] || fail 'convergence health ran after committed rollback recovery' +[[ ! -e "$recovery_transaction" ]] || fail 'committed rollback recovery retained its transaction' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# An unsafe rollback commit marker type must block before recovery mutation. +recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.marker-type +mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" +mkdir "$recovery_transaction/application-rollback-committed" +units_before_marker=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum) +policy_before_marker=$(sha256sum "$root/var/lib/ci-fleet-deployer/active-policy.conf") +expect_failure 'application rollback commit marker is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_marker == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum)" && $policy_before_marker == "$(sha256sum "$root/var/lib/ci-fleet-deployer/active-policy.conf")" ]] || fail 'directory rollback marker mutated managed state' +[[ -d "$recovery_transaction" ]] || fail 'blocked rollback marker transaction was discarded' +rmdir "$recovery_transaction/application-rollback-committed" +ln -s "$tmp/missing-rollback-target" "$recovery_transaction/application-rollback-committed" +expect_failure 'application rollback commit marker is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_marker == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum)" && $policy_before_marker == "$(sha256sum "$root/var/lib/ci-fleet-deployer/active-policy.conf")" ]] || fail 'broken-symlink rollback marker mutated managed state' +rm -rf -- "$recovery_transaction" +expect_success "$installer" --check --config "$config" >/dev/null + +# Finalizing a committed rollback must publish over an unusable incumbent deployed snapshot. +deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") +cp "$deployed_dir/policy.conf" "$tmp/incumbent-policy.saved" +python3 - "$deployed_dir/policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ADAPTER_SHA256=', 'ADAPTER_SHA256=' + 'e'*64 + '\n#', 1)) +PY +recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.unusable-incumbent +mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$recovery_transaction/state/$name"; printf '%s\n' "$name" >>"$recovery_transaction/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$recovery_transaction/current-target" +install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" +recovery=$(expect_success "$installer" --upgrade --config "$config") +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$recovery" || fail 'unusable-incumbent recovery lacks the operator reconciliation action' +[[ ! -e "$recovery_transaction" ]] || fail 'unusable-incumbent recovery retained its transaction' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A deployed rollback pair whose adapter bytes no longer match its recorded digest must not promote. +# Rebuild the retained pair consumed by the unusable-incumbent recovery fixture. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' +deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") +deployed_target=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") +cp "$deployed_dir/policy.conf" "$tmp/deployed-policy.saved" +python3 - "$deployed_dir/policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ADAPTER_SHA256=', 'ADAPTER_SHA256=' + 'f'*64 + '\n#', 1)) +PY +if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_before_digest=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); else lkg_before_digest=absent; fi +expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --upgrade --config "$config" >/dev/null +lkg_after_digest=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_digest=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_digest == "$lkg_after_digest" ]] || fail 'digest-mismatched deployed pair replaced last-known-good' +if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'blocked digest promotion left a recovery transaction'; fi +# Rollback intentionally publishes the retained pair over the unusable +# incumbent snapshot: the drifted deployed pair is retired, never promoted. +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$rollback" || fail 'drifted-incumbent rollback lacks the operator reconciliation action' +[[ ! -e "$deployed_dir" ]] || fail 'drifted deployed snapshot survived its rollback retirement' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +# Rebuild the retained pair consumed by the drifted-incumbent rollback. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' + +# A deployed rollback pair whose credential reference drifted must not promote. +deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") +cp "$deployed_dir/policy.conf" "$tmp/deployed-policy-cred.saved" +python3 - "$deployed_dir/policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external').replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=external:bad')) +PY +if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_before_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); else lkg_before_credential=absent; fi +expect_failure 'deployed rollback policy has an invalid external secret-manager adapter reference' "$installer" --upgrade --config "$config" >/dev/null +lkg_after_credential=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_credential == "$lkg_after_credential" ]] || fail 'credential-drifted deployed pair replaced last-known-good' +# Rollback intentionally publishes the retained pair over the unusable +# incumbent snapshot: the credential-drifted pair is retired, never promoted. +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$rollback" || fail 'credential-drifted rollback lacks the operator reconciliation action' +[[ ! -e "$deployed_dir" ]] || fail 'credential-drifted deployed snapshot survived its rollback retirement' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# Finalizing a committed rollback with drifted adapter bytes must not publish or delete retained state. +recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.drifted-adapter +rm -rf "$root/var/lib/ci-fleet-deployer/deployed" +mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$recovery_transaction/state/$name"; printf '%s\n' "$name" >>"$recovery_transaction/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$recovery_transaction/current-target" +install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" +cp "$adapter" "$tmp/adapter.saved" +printf '# drifted\n' >>"$adapter" +if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_before_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); else lkg_before_finalize=absent; fi +expect_failure 'adapter digest does not match the protected regular file' "$installer" --repair --config "$config" >/dev/null +[[ -d "$recovery_transaction" ]] || fail 'failed finalize discarded its recovery transaction' +lkg_after_finalize=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +[[ $lkg_before_finalize == "$lkg_after_finalize" ]] || fail 'failed finalize deleted the retained rollback pair' +cat "$tmp/adapter.saved" >"$adapter" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A symlinked systemd boundary must block before transaction recovery deletes through it. +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +boundary_tx=$root/var/lib/ci-fleet-deployer/.transaction.boundary +mkdir -m 0700 "$boundary_tx" "$boundary_tx/units" "$boundary_tx/state" +mv "$root/etc/systemd/system" "$tmp/systemd.real" +ln -s "$tmp/systemd.real" "$root/etc/systemd/system" +printf 'decoy\n' >"$tmp/systemd.real/ci-fleet-deployer.service" +expect_failure 'systemd unit directory has an unsafe owner or type' "$installer" --repair --config "$config" >/dev/null +[[ $(<"$tmp/systemd.real/ci-fleet-deployer.service") == decoy ]] || fail 'recovery deleted through a symlinked systemd boundary' +[[ -d "$boundary_tx" ]] || fail 'blocked boundary recovery discarded its transaction' +rm "$root/etc/systemd/system" +mv "$tmp/systemd.real" "$root/etc/systemd/system" +rm "$root/etc/systemd/system/ci-fleet-deployer.service" +rm -rf -- "$boundary_tx" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A group- or world-writable systemd boundary must block before transaction backups. +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +chmod 0777 "$root/etc/systemd/system" +units_before_writable=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) +expect_failure 'systemd unit directory is group- or world-writable' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_writable == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'writable systemd boundary allowed a transaction backup' +boundary_tx=$root/var/lib/ci-fleet-deployer/.transaction.writable-recovery +mkdir -m 0700 "$boundary_tx" "$boundary_tx/units" "$boundary_tx/state" +expect_failure 'systemd unit directory is group- or world-writable' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_writable == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'writable systemd boundary allowed a recovery restore' +rm -rf -- "$boundary_tx" +chmod 0755 "$root/etc/systemd/system" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A symlinked install root must block before recovery restores pointers. +boundary_tx=$root/var/lib/ci-fleet-deployer/.transaction.install-boundary +mkdir -m 0700 "$boundary_tx" "$boundary_tx/units" "$boundary_tx/state" +mv "$root/opt/ci-fleet-deployer" "$tmp/install-root.real" +ln -s "$tmp/install-root.real" "$root/opt/ci-fleet-deployer" +expect_failure 'unsafe symlinked managed directory' "$installer" --repair --config "$config" >/dev/null +[[ -L "$tmp/install-root.real/current" ]] || fail 'symlinked install boundary mutated the activation pointer' +rm "$root/opt/ci-fleet-deployer" +mv "$tmp/install-root.real" "$root/opt/ci-fleet-deployer" +rm -rf -- "$boundary_tx" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A managed-unit drop-in must break convergence until removed. +mkdir "$root/etc/systemd/system/ci-fleet-deployer.service.d" +printf '[Service]\nExecStart=\n' >"$root/etc/systemd/system/ci-fleet-deployer.service.d/override.conf" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +expect_failure 'has an unreviewed drop-in override' "$installer" --repair --config "$config" >/dev/null +[[ -f "$root/etc/systemd/system/ci-fleet-deployer.service.d/override.conf" ]] || fail 'repair discarded an unmanaged drop-in' +rm -rf "$root/etc/systemd/system/ci-fleet-deployer.service.d" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A group- or world-writable managed unit must block before transaction backup. +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" +units_before_mode=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) +expect_failure 'managed systemd unit has an unsafe owner, mode, or type' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_mode == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'writable unit allowed a transaction backup' +chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# Rebuild a retained rollback pair consumed by the drifted-incumbent rollback +# and finalize recovery fixtures. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' + +# A lost install state with surviving release/units must not be treated as a fresh install. +mv "$root/var/lib/ci-fleet-deployer/install-state.json" "$tmp/install-state.saved" +expect_failure 'restore install state before convergence' "$installer" --repair --config "$config" >/dev/null +mv "$tmp/install-state.saved" "$root/var/lib/ci-fleet-deployer/install-state.json" +chmod 0600 "$root/var/lib/ci-fleet-deployer/install-state.json" +expect_success "$installer" --repair --config "$config" >/dev/null + +# Lost install state with only a retained deployed snapshot is still an installation. +mv "$root/var/lib/ci-fleet-deployer/install-state.json" "$tmp/install-state.saved" +rm -f "$root/opt/ci-fleet-deployer/current" +for stale_unit in "$root"/etc/systemd/system/ci-fleet-deployer*; do rm -f "$stale_unit"; done +expect_failure 'restore install state before convergence' "$installer" --repair --config "$config" >/dev/null +mv "$tmp/install-state.saved" "$root/var/lib/ci-fleet-deployer/install-state.json" +chmod 0600 "$root/var/lib/ci-fleet-deployer/install-state.json" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# Rollback must work from the retained pair without a usable candidate config. +mv "$config" "$tmp/config.saved" +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'config-independent rollback did not report change' +mv "$tmp/config.saved" "$config"; chmod 0600 "$config" +expect_success "$installer" --repair --config "$config" >/dev/null +# Rebuild the retained pair consumed by the config-independent rollback. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' +printf 'malformed line\n' >"$config"; chmod 0600 "$config" +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'malformed-config rollback did not report change' +expect_success "$installer" --uninstall --config "$config" >/dev/null +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'malformed-config uninstall retained the activation pointer' +write_config +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --install --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# rollback_available must reflect a validated retained pair. +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'expected a retained rollback point' +chmod 0644 "$root/var/lib/ci-fleet-deployer/last-known-good.json" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'drifted retained pair still reported rollback_available=yes' +chmod 0600 "$root/var/lib/ci-fleet-deployer/last-known-good.json" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=yes' <<<"$available_check" || fail 'valid retained pair was not reported rollback_available=yes' +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ADAPTER_SHA256=', 'ADAPTER_SHA256=' + 'f'*64 + '\n#', 1)) +PY +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'digest-mismatched retained adapter still reported rollback_available=yes' +install -m 0600 "$tmp/lkg-policy.saved" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" 2>/dev/null || true +# A retained pair whose release is missing must report rollback_available=no. +missing_core=cccccccccccccccccccccccccccccccccccccccc +cp "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$tmp/lkg-policy-release.saved" +cp "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$tmp/lkg-state-release.saved" +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$missing_core" <<'PY' +from pathlib import Path +import json, re, sys +policy, state, missing = sys.argv[1], sys.argv[2], sys.argv[3] +p = Path(policy); p.write_text(re.sub(r'(?m)^CORE_REF=[0-9a-f]{40}$', 'CORE_REF=' + missing, p.read_text())) +s = json.loads(Path(state).read_text()); s['core_ref'] = missing +Path(state).write_text(json.dumps(s, indent=2, sort_keys=True) + '\n') +PY +chmod 0600 "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$root/var/lib/ci-fleet-deployer/last-known-good.json" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'release-missing retained pair still reported rollback_available=yes' +install -m 0600 "$tmp/lkg-policy-release.saved" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" +install -m 0600 "$tmp/lkg-state-release.saved" "$root/var/lib/ci-fleet-deployer/last-known-good.json" +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +write_production_gate +write_evidence production example-production +write_config production example-production +expect_failure 'installed environment and target identity cannot change in place' "$installer" --upgrade --config "$config" >/dev/null +write_evidence +write_config staging example-staging + +active=$root/var/lib/ci-fleet-deployer/active-operation +printf 'unrelated-active\n' >"$tmp/unrelated-active" +ln -s "$tmp/unrelated-active" "$active" +expect_failure 'active operation marker is an unsafe symlink' "$installer" --repair --config "$config" >/dev/null +rm "$active" +printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$active" +chmod 0600 "$active" +CI_FLEET_DEPLOYER_TEST_LIVE_PID=$$ expect_failure 'active deployment prevents this operation' "$installer" --upgrade --config "$config" >/dev/null +CI_FLEET_DEPLOYER_TEST_LIVE_PID=$$ expect_failure 'active deployment prevents drain' "$installer" --drain --config "$config" >/dev/null +rm "$active" +drain=$(expect_success "$installer" --drain --config "$config") +grep -Fq 'result=CHANGED' <<<"$drain" || fail 'drain marker was not created' +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'drain state is absent' +mv "$fake_bin/docker" "$fake_bin/docker.unavailable" +expect_success "$installer" --drain --config "$config" >/dev/null +mv "$fake_bin/docker.unavailable" "$fake_bin/docker" +chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'installed deployer state is absent or drifted; repair before resume' "$installer" --resume --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'failed resume removed drain state' +chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'repair implicitly resumed the deployer' +resume=$(expect_success "$installer" --resume --config "$config") +grep -Fq 'result=CHANGED' <<<"$resume" || fail 'resume did not clear drain state' +grep -Fq 'health=healthy' <<<"$resume" || fail 'resume report omitted verified health' +[[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'resume retained drain state' +chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'installed deployer state is absent or drifted; repair before resume' "$installer" --resume --config "$config" >/dev/null +chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_success "$installer" --repair --config "$config" >/dev/null +repeat_resume=$(expect_success "$installer" --resume --config "$config") +grep -Fq 'result=NO_CHANGE' <<<"$repeat_resume" || fail 'repeated resume was not idempotent' +expect_success "$installer" --drain --config "$config" >/dev/null +rm "$root/var/lib/ci-fleet-deployer/drained" +printf 'unrelated-drain-target\n' >"$tmp/unrelated-drain-target" +ln -s "$tmp/unrelated-drain-target" "$root/var/lib/ci-fleet-deployer/drained" +expect_failure 'drain marker must be a regular file, not a symlink' "$installer" --uninstall --config "$config" >/dev/null +[[ $(<"$tmp/unrelated-drain-target") == unrelated-drain-target ]] || fail 'uninstall followed an unsafe drain marker' +rm "$root/var/lib/ci-fleet-deployer/drained" +install -m 0600 /dev/null "$root/var/lib/ci-fleet-deployer/drained" + +unrelated=$root/var/lib/ci-fleet-deployer/operator-note +printf 'preserve\n' >"$unrelated" +mv "$root/etc/ci-fleet-deployer/adapters" "$root/etc/ci-fleet-deployer/adapters.retained" +mv "$root/etc/ci-fleet-deployer/credentials" "$root/etc/ci-fleet-deployer/credentials.retained" +mv "$root/etc/ci-fleet-deployer/evidence" "$root/etc/ci-fleet-deployer/evidence.retained" +export FAKE_DOCKER_INFO_EXIT=1 +uninstall=$(expect_success "$installer" --uninstall --config "$config") +unset FAKE_DOCKER_INFO_EXIT +mv "$root/etc/ci-fleet-deployer/adapters.retained" "$root/etc/ci-fleet-deployer/adapters" +mv "$root/etc/ci-fleet-deployer/credentials.retained" "$root/etc/ci-fleet-deployer/credentials" +mv "$root/etc/ci-fleet-deployer/evidence.retained" "$root/etc/ci-fleet-deployer/evidence" +grep -Fq 'result=CHANGED' <<<"$uninstall" || fail 'uninstall did not report change' +[[ -f "$config" && -f "$credential" && -f "$unrelated" ]] || fail 'uninstall removed retained operator state or credentials' +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'uninstall retained activation pointer' +repeat_uninstall=$(expect_success "$installer" --uninstall --config "$config") +grep -Fq 'result=NO_CHANGE' <<<"$repeat_uninstall" || fail 'repeated uninstall was not idempotent' + +# A drifted managed unit directory must fail closed before any uninstall mutation. +expect_success "$installer" --install --config "$config" >/dev/null +rm "$root/etc/systemd/system/ci-fleet-deployer-drain.service" +mkdir "$root/etc/systemd/system/ci-fleet-deployer-drain.service" +uninstall_before=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) +[[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'test setup expected no drain marker before drifted uninstall' +expect_failure 'managed unit ci-fleet-deployer-drain.service has an unsafe type' "$installer" --uninstall --config "$config" >/dev/null +[[ "$uninstall_before" == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'unsafe unit type partially mutated the host during uninstall' +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'failed closed uninstall removed the activation pointer' +[[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'failed closed uninstall created the drain marker' +rmdir "$root/etc/systemd/system/ci-fleet-deployer-drain.service" +expect_success "$installer" --uninstall --config "$config" >/dev/null +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after correcting unit drift did not uninstall' + +# A symlinked install root must block uninstall before its target's pointer is touched. +expect_success "$installer" --install --config "$config" >/dev/null +mv "$root/opt/ci-fleet-deployer" "$tmp/install-root-uninstall.real" +ln -s "$tmp/install-root-uninstall.real" "$root/opt/ci-fleet-deployer" +expect_failure 'unsafe symlinked managed directory' "$installer" --uninstall --config "$config" >/dev/null +[[ -L "$tmp/install-root-uninstall.real/current" ]] || fail 'uninstall removed a pointer through a symlinked install root' +rm "$root/opt/ci-fleet-deployer" +mv "$tmp/install-root-uninstall.real" "$root/opt/ci-fleet-deployer" +expect_success "$installer" --uninstall --config "$config" >/dev/null +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after correcting install-root drift did not uninstall' + +# Uninstall must fail when systemd cannot reload after unit removal. +expect_success "$installer" --install --config "$config" >/dev/null +FAKE_SYSTEMCTL_FAIL_COMMAND=daemon-reload expect_failure 'systemd manager reload failed after unit removal' "$installer" --uninstall --config "$config" >/dev/null +unset FAKE_SYSTEMCTL_FAIL_COMMAND +expect_success "$installer" --uninstall --config "$config" >/dev/null +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after reload failure did not uninstall' + +# Uninstall must still remove the deployment surface when the configuration directory is absent. +expect_success "$installer" --install --config "$config" >/dev/null +mv "$root/etc/ci-fleet-deployer" "$tmp/etc-deployer.saved" +uninstall=$(expect_success "$installer" --uninstall --config "$config") +grep -Fq 'result=CHANGED' <<<"$uninstall" || fail 'uninstall without configuration did not report change' +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'uninstall without configuration retained the activation pointer' +if compgen -G "$root/etc/systemd/system/ci-fleet-deployer*" >/dev/null; then fail 'uninstall without configuration retained managed units'; fi +mv "$tmp/etc-deployer.saved" "$root/etc/ci-fleet-deployer" + +# Uninstall must stop a timer whose unit file has drifted away. +expect_success "$installer" --install --config "$config" >/dev/null +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +FAKE_SYSTEMCTL_LOG=$tmp/systemctl-drift.log expect_success "$installer" --uninstall --config "$config" >/dev/null +grep -Fq 'disable --now ci-fleet-deployer-cleanup.timer' "$tmp/systemctl-drift.log" || fail 'uninstall left a loaded drifted timer running' +if compgen -G "$root/etc/systemd/system/ci-fleet-deployer*" >/dev/null; then fail 'drifted-timer uninstall retained managed units'; fi + +# An untracked symlink in the deployer unit source must block checkout validation. +expect_success "$installer" --install --config "$config" >/dev/null +ln -s "$credential" "$repo_root/deploy/deployer/leak" +expect_failure 'deployer unit source contains an unsafe or untracked entry' "$installer" --repair --config "$config" >/dev/null +[[ $(stat -c %a "$credential") == 600 ]] || fail 'untracked checkout symlink exposed credential bytes' +rm "$repo_root/deploy/deployer/leak" +printf 'unreviewed\n' >"$repo_root/deploy/deployer/extra-unit.service" +expect_failure 'checkout snapshot contains unreviewed entries' "$installer" --repair --config "$config" >/dev/null +rm "$repo_root/deploy/deployer/extra-unit.service" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A retained policy naming a credential outside the protected directory, or a +# malformed external reference, must report rollback_available=no. +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" ]] || fail 'test setup expected a retained policy' +cp "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$tmp/lkg-policy.saved" +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external').replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=external:bad')) +PY +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'malformed external retained credential still reported rollback_available=yes' +install -m 0600 "$tmp/lkg-policy.saved" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=/etc/hostname')) +PY +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'out-of-directory retained credential still reported rollback_available=yes' +install -m 0600 "$tmp/lkg-policy.saved" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=yes' <<<"$available_check" || fail 'restored retained pair was not reported rollback_available=yes' + +# Retained-transaction recovery must validate the systemd boundary before stopping timers. +expect_success "$installer" --uninstall --config "$config" >/dev/null +expect_success "$installer" --install --config "$config" >/dev/null +boundary_recovery=$root/var/lib/ci-fleet-deployer/.transaction.writable-boundary +mkdir -m 0700 "$boundary_recovery" "$boundary_recovery/units" "$boundary_recovery/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$boundary_recovery/state/$name"; printf '%s\n' "$name" >>"$boundary_recovery/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$boundary_recovery/current-target" +printf '%s\n' ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer >"$boundary_recovery/timers-enabled" +chmod 0777 "$root/etc/systemd/system" +expect_failure 'systemd unit directory is group- or world-writable' "$installer" --repair --config "$config" >/dev/null +[[ $(stat -c %a "$root/etc/systemd/system") == 777 ]] || fail 'writable boundary recovery mutated the unit directory' +rm -rf -- "$boundary_recovery" +chmod 0755 "$root/etc/systemd/system" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# Retained-transaction recovery must validate the deployed-state boundary before pointer mutation. +boundary_recovery=$root/var/lib/ci-fleet-deployer/.transaction.deployed-symlink +mkdir -m 0700 "$boundary_recovery" "$boundary_recovery/units" "$boundary_recovery/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$boundary_recovery/state/$name"; printf '%s\n' "$name" >>"$boundary_recovery/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$boundary_recovery/current-target" +printf 'absent\n' >"$boundary_recovery/deployed-target" +mv "$root/var/lib/ci-fleet-deployer/deployed" "$tmp/deployed.real" +mkdir "$tmp/deployed.real-target"; ln -s "$tmp/deployed.real-target" "$root/var/lib/ci-fleet-deployer/deployed" +printf 'decoy\n' >"$tmp/deployed.real-target/current" +expect_failure 'deployed snapshot directory is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ $(<"$tmp/deployed.real-target/current") == decoy ]] || fail 'recovery mutated state through a symlinked deployed directory' +[[ -d "$boundary_recovery" ]] || fail 'blocked deployed-boundary recovery discarded its transaction' +rm "$root/var/lib/ci-fleet-deployer/deployed"; mv "$tmp/deployed.real" "$root/var/lib/ci-fleet-deployer/deployed" +rm -rf -- "$boundary_recovery" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A failed timer shutdown must fail the uninstall before any unit removal. +expect_success "$installer" --install --config "$config" >/dev/null +uninstall_before=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) +FAKE_SYSTEMCTL_FAIL_COMMAND=disable expect_failure 'deployer timers did not stop during uninstall' "$installer" --uninstall --config "$config" >/dev/null +[[ "$uninstall_before" == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'failed timer shutdown partially uninstalled units' +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'failed timer shutdown removed the activation pointer' +expect_success "$installer" --uninstall --config "$config" >/dev/null +repeat_uninstall=$(expect_success "$installer" --uninstall --config "$config") +grep -Fq 'result=NO_CHANGE' <<<"$repeat_uninstall" || fail 'repeated uninstall was not idempotent after timer removal' + +# Runtime contract: exact-head request/evidence, drain and scoped adapter calls. +write_evidence staging example-staging +write_config staging example-staging +python3 - "$config" "$capability" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=manual-exact-head', 'APPROVAL_PROVIDER=github-environment') + 'APPROVAL_CAPABILITY_EVIDENCE_PATH='+sys.argv[2]+'\n') +PY +expect_success "$installer" --install --config "$config" >/dev/null +request=$root/var/lib/ci-fleet-deployer/request.conf +cp "$approval" "$request" +chmod 0600 "$request" +export CI_FLEET_DEPLOYER_CONFIG=$config CI_FLEET_DEPLOYER_REQUEST=$request +# Production deployment remains separately gated: the runtime must reject it. +cp "$config" "$tmp/config.staging" +write_production_gate +write_evidence production example-production +write_config production example-production +python3 - "$request" "$approval" <<'PY' +from pathlib import Path +import sys +for name in sys.argv[1:]: + p=Path(name); p.write_text(p.read_text().replace('ENVIRONMENT=staging', 'ENVIRONMENT=production').replace('TARGET_ID=example-staging', 'TARGET_ID=example-production')) +PY +expect_failure 'production deployment is not authorized by the current accepted scope' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == 0 ]] || fail 'gated production path reached the deployment adapter' +install -m 0600 "$tmp/config.staging" "$config" +write_evidence staging example-staging +cp "$approval" "$request"; chmod 0600 "$request" +expect_failure 'usage: deployer-runtime.sh health|cleanup|deploy|drain' "$runtime" rollback >/dev/null +mv "$adapter" "$adapter.saved" +expect_success "$runtime" drain >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'runtime drain required a healthy adapter' +rm "$root/var/lib/ci-fleet-deployer/drained" +mv "$adapter.saved" "$adapter" +export FAKE_ADAPTER_REPLACE_PATH=$adapter +expect_success "$runtime" health >/dev/null +unset FAKE_ADAPTER_REPLACE_PATH +rm "$adapter"; mv "$adapter.saved" "$adapter" +expect_success "$runtime" health >/dev/null +expect_success "$runtime" cleanup >/dev/null +adapter_config=$tmp/adapter-config +FAKE_ADAPTER_RECORD_CONFIG=$adapter_config expect_success "$runtime" health >/dev/null +unset FAKE_ADAPTER_RECORD_CONFIG +[[ -e "$adapter_config.seen" ]] || fail 'adapter config snapshot was not recorded' +[[ $(<"$adapter_config") == "$root"/var/lib/ci-fleet-deployer/.active-policy.* ]] || fail 'adapter did not receive an immutable policy snapshot path' +cmp -s "$adapter_config.content" "$root/var/lib/ci-fleet-deployer/active-policy.conf" || fail 'adapter policy snapshot content differs from the validated policy' +compgen -G "$root/var/lib/ci-fleet-deployer/.active-policy.*" >/dev/null && fail 'policy snapshot was not cleaned up' +chmod 0644 "$credential" +cleanup_calls_before=$(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) +expect_failure 'credential file has unsafe owner or mode' "$runtime" cleanup >/dev/null +[[ $(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) == "$cleanup_calls_before" ]] || fail 'cleanup ran with unsafe file credentials' +chmod 0600 "$credential" +mv "$root/etc/ci-fleet-deployer/credentials" "$root/etc/ci-fleet-deployer/credentials.real" +ln -s "$root/etc/ci-fleet-deployer/credentials.real" "$root/etc/ci-fleet-deployer/credentials" +expect_failure 'credential directory has unsafe owner, mode, or type' "$runtime" cleanup >/dev/null +rm "$root/etc/ci-fleet-deployer/credentials" +mv "$root/etc/ci-fleet-deployer/credentials.real" "$root/etc/ci-fleet-deployer/credentials" +mkdir "$root/var/lib/ci-fleet-deployer/.transaction.interrupted" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'interrupted installer transaction requires recovery' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran during an interrupted installer transaction' +rmdir "$root/var/lib/ci-fleet-deployer/.transaction.interrupted" +printf 'pid=999999\nstarted_at=1\n' >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'active operation marker requires recovery' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment replaced an unresolved active marker' +rm "$root/var/lib/ci-fleet-deployer/active-operation" +printf 'runner\n' >"$root/etc/systemd/system/actions.runner.late-added.service" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +cleanup_calls_before=$(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) +health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) +expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" deploy >/dev/null +expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" cleanup >/dev/null +expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" health >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran after role isolation drift' +[[ $(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) == "$cleanup_calls_before" ]] || fail 'cleanup ran after role isolation drift' +[[ $(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) == "$health_calls_before" ]] || fail 'health adapter ran after role isolation drift' +rm "$root/etc/systemd/system/actions.runner.late-added.service" +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=github-environment', 'APPROVAL_PROVIDER=github-environmnt')) +PY +expect_failure 'unsupported approval provider' "$runtime" deploy >/dev/null +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('SCHEMA_VERSION=1', 'SCHEMA_VERSION=2', 1)) +PY +expect_failure 'configuration has an unsupported or missing schema version' "$runtime" deploy >/dev/null +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('SCHEMA_VERSION=2', 'SCHEMA_VERSION=1', 1)) +PY +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text('\n'.join(x for x in p.read_text().splitlines() if not x.startswith('CORE_REF='))+'\n') +PY +expect_failure 'configuration is missing a valid core revision' "$runtime" deploy >/dev/null +python3 - "$config" "$core_ref" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text() + 'CORE_REF='+sys.argv[2]+'\n') +PY +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=github-environmnt', 'APPROVAL_PROVIDER=external-exact-head').replace('APPROVAL_CAPABILITY_EVIDENCE_PATH='+str(p.parent/'evidence/github-capability.conf'), '').rstrip()+'\n') +PY +cp "$approval" "$request"; chmod 0600 "$request" +expect_success "$runtime" deploy >/dev/null +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=approval-20260808-2').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:01:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=external-exact-head', 'APPROVAL_PROVIDER=github-environment') + 'APPROVAL_CAPABILITY_EVIDENCE_PATH='+str(p.parent/'evidence/github-capability.conf')+'\n') +PY +python3 - "$approval" "$request" <<'PY' +from pathlib import Path +import sys +for name in sys.argv[1:]: + p=Path(name); p.write_text(p.read_text().replace('APPROVED_AT=2026-08-08T20:01:00Z', 'APPROVED_AT=2026-99-99T99:99:99Z')) +PY +expect_failure 'deployment request has an invalid approval time' "$runtime" deploy >/dev/null +compgen -G "$root/var/lib/ci-fleet-deployer/.active-policy.*" >/dev/null && fail 'rejected request left a stale policy snapshot' +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=approval-20260808-3').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:02:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +python3 - "$request" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-3', 'APPROVAL_ID=forged-approval')) +PY +expect_failure 'deployment request does not match protected approval APPROVAL_ID' "$runtime" deploy >/dev/null +cp "$approval" "$request"; chmod 0600 "$request" +mv "$checkpoint" "$checkpoint.saved" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'checkpoint evidence must be a regular file' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran without checkpoint evidence' +mv "$checkpoint.saved" "$checkpoint" +mv "$capability" "$capability.saved" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'GitHub capability evidence must be a regular file' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran without GitHub capability evidence' +mv "$capability.saved" "$capability" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +# A signal arriving after adapter success must not abort publication: the +# approval is consumed and the application change is applied, so the runtime +# masks INT/TERM until the new snapshot pointer and audit record are durable. +deployed_before_signal=$(readlink "$deployed_current") +CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF=TERM expect_success "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == $((deploy_calls_before + 1)) ]] || fail 'signaled runtime did not reach the adapter' +[[ -L "$deployed_current" && $(readlink "$deployed_current") != "$deployed_before_signal" ]] || fail 'masked signal prevented deployed pointer publication' +[[ ! -e "$active" ]] || fail 'completed deployment left the active operation marker' +if compgen -G "$root/var/lib/ci-fleet-deployer/.active.*" >/dev/null; then fail 'completed deployment left an unpublished active marker temporary'; fi +rm -f "$root/var/lib/ci-fleet-deployer/last-request.conf" +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" +cp "$approval" "$request"; chmod 0600 "$request" +deployed_target=$(readlink "$deployed_current") +rm "$deployed_current" +expect_failure 'deployed rollback snapshot is missing' "$runtime" deploy >/dev/null +ln -s "$deployed_target" "$deployed_current" +rm "$deployed_current" +ln -s "$root/var/lib/ci-fleet-deployer/deployed/$deployed_target" "$deployed_current" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'deployed snapshot pointer target is not canonical' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment consumed approval with a noncanonical deployed pointer' +[[ ! -e "$active" ]] || fail 'rejected deployment left the active operation marker' +rm "$deployed_current" +ln -s "$deployed_target" "$deployed_current" +chmod 0644 "$credential" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'credential file has unsafe owner or mode' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran with unsafe file credentials' +chmod 0600 "$credential" +printf 'unrelated-audit\n' >"$tmp/unrelated-audit" +rm -f "$root/var/log/ci-fleet-deployer/audit.log" +ln -s "$tmp/unrelated-audit" "$root/var/log/ci-fleet-deployer/audit.log" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" && $(<"$tmp/unrelated-audit") == unrelated-audit ]] || fail 'unsafe audit storage was touched after adapter execution' +rm "$root/var/log/ci-fleet-deployer/audit.log" +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=failed-adapter-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:03:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_FAIL=$tmp/fail-adapter +printf 'deploy\n' >"$FAKE_ADAPTER_FAIL" +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-adapter" +grep -Fq 'approval=failed-adapter-attempt approver=example-reviewer policy=example-staging-policy-v1 checkpoint=checkpoint-20260808-1 authorized_by=none gate=none result=failed phase=adapter status=42' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'consumed failed deployment was not audited' +rm -f "$root/var/lib/ci-fleet-deployer/last-request.conf" +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=snapshot-mutation-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:04:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT=$root/var/lib/ci-fleet-deployer/deployed +expect_failure 'prepared deployed snapshot changed during deployment' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT +grep -Fq 'approval=snapshot-mutation-attempt approver=example-reviewer policy=example-staging-policy-v1 checkpoint=checkpoint-20260808-1 authorized_by=none gate=none result=failed phase=post-adapter status=2' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'post-adapter deployment failure was not audited with its real status' +[[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == 1 ]] || fail 'failed publication leaked an unreachable prepared snapshot' +[[ -e "$deployed_current" ]] || fail 'failed publication dangled the deployed pointer' + +# A signal while the adapter runs must remove only the unreachable prepared snapshot. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=signal-mid-adapter-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:04:30Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_SIGNAL_PPID=1 +if "$runtime" deploy >/dev/null 2>&1; then fail 'signaled deployment reported success'; fi +unset FAKE_ADAPTER_SIGNAL_PPID +[[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == 1 ]] || fail 'interrupted deployment leaked an unreachable prepared snapshot' +[[ -e "$deployed_current" ]] || fail 'interrupted deployment dangled the deployed pointer' +[[ ! -e "$active" ]] || fail 'interrupted deployment left the active operation marker' +grep -Fq 'approval=signal-mid-adapter-attempt' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'interrupted deployment was not audited as failed' +rm -f "$root/var/lib/ci-fleet-deployer/last-request.conf" +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=audit-replacement-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_MUTATE_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_MUTATE_AUDIT_TARGET=$tmp/unrelated-audit +expect_failure 'deployer audit log changed during deployment' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_MUTATE_AUDIT_PATH FAKE_ADAPTER_MUTATE_AUDIT_TARGET +[[ $(<"$tmp/unrelated-audit") == unrelated-audit ]] || fail 'adapter audit replacement redirected the trusted append' +rm "$root/var/log/ci-fleet-deployer/audit.log" + +# An adapter that replaces the audit log must not lose the deployment record. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=audit-unlink-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:30Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_MUTATE_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_MUTATE_AUDIT_MODE=unlink +expect_failure 'deployer audit log changed during deployment' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_MUTATE_AUDIT_PATH FAKE_ADAPTER_MUTATE_AUDIT_MODE +grep -Fq 'approval=audit-unlink-attempt' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'adapter audit replacement lost the deployment audit record' +[[ $(<"$root/var/log/ci-fleet-deployer/audit.log") != *adapter-replacement* ]] || fail 'adapter replacement content entered the trusted audit log' +rm "$root/var/log/ci-fleet-deployer/audit.log" + +# An adapter that deletes its consumption marker and fails must not enable replay. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=marker-delete-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:45Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +printf 'deploy\n' >"$tmp/fail-deploy-marker"; export FAKE_ADAPTER_FAIL=$tmp/fail-deploy-marker +export FAKE_ADAPTER_DELETE_CONSUMED_GLOB="$root/var/lib/ci-fleet-deployer/consumed-requests/*" +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_FAIL FAKE_ADAPTER_DELETE_CONSUMED_GLOB; rm "$tmp/fail-deploy-marker" +if ! compgen -G "$root/var/lib/ci-fleet-deployer/consumed-requests/*" >/dev/null; then fail 'adapter-deleted consumption marker was not restored'; fi +cp "$approval" "$request"; chmod 0600 "$request" +expect_failure 'deployment request was already consumed' "$runtime" deploy >/dev/null +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" + +# An adapter that deletes the whole consumption directory and fails must not enable replay. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=dir-delete-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:50Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +printf 'deploy\n' >"$tmp/fail-deploy-dir"; export FAKE_ADAPTER_FAIL=$tmp/fail-deploy-dir +export FAKE_ADAPTER_DELETE_CONSUMED_GLOB="$root/var/lib/ci-fleet-deployer/consumed-requests" +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_FAIL FAKE_ADAPTER_DELETE_CONSUMED_GLOB; rm "$tmp/fail-deploy-dir" +if ! compgen -G "$root/var/lib/ci-fleet-deployer/consumed-requests/*" >/dev/null; then fail 'adapter-deleted consumption directory was not restored with its marker'; fi +cp "$approval" "$request"; chmod 0600 "$request" +expect_failure 'deployment request was already consumed' "$runtime" deploy >/dev/null +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" + +# A failing adapter cannot leave recovery state or the audit log with unsafe metadata. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=metadata-drift-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:55Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +incumbent_dir=$(readlink -f "$deployed_current") +printf 'deploy\n' >"$tmp/fail-after-metadata" +export FAKE_ADAPTER_MUTATE_INCUMBENT_PATH=$incumbent_dir +export FAKE_ADAPTER_MUTATE_LKG_ROOT=$root/var/lib/ci-fleet-deployer +export FAKE_ADAPTER_CHMOD_DURING=$root/var/log/ci-fleet-deployer/audit.log +export FAKE_ADAPTER_CHOWN_DURING=$root/var/log/ci-fleet-deployer/audit.log +export FAKE_ADAPTER_FAIL_AFTER_MUTATION=$tmp/fail-after-metadata +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_MUTATE_INCUMBENT_PATH FAKE_ADAPTER_MUTATE_LKG_ROOT FAKE_ADAPTER_CHMOD_DURING FAKE_ADAPTER_CHOWN_DURING FAKE_ADAPTER_FAIL_AFTER_MUTATION +expected_uid=$(id -u) +[[ $(stat -c '%u:%a' "$incumbent_dir") == "$expected_uid:700" ]] || fail 'failed adapter left incumbent directory metadata unsafe' +for file in "$incumbent_dir"/policy.conf "$incumbent_dir"/state.json "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$root/var/log/ci-fleet-deployer/audit.log"; do + [[ $(stat -c '%u:%a' "$file") == "$expected_uid:600" ]] || fail "failed adapter left recovery metadata unsafe: $file" +done +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" + +# A successful adapter also cannot leave the retained rollback pair unsafe. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=lkg-metadata-drift-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:56Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +FAKE_ADAPTER_MUTATE_LKG_ROOT=$root/var/lib/ci-fleet-deployer expect_success "$runtime" deploy >/dev/null +for file in "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf"; do + [[ $(stat -c '%u:%a' "$file") == "$expected_uid:600" ]] || fail "successful adapter left retained rollback metadata unsafe: $file" +done + +# Signal at the deployed-snapshot publication boundary must not delete the published snapshot. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=signal-at-publication-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:06:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_MUTATE_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_MUTATE_AUDIT_TARGET=$tmp/unrelated-audit +expect_failure 'deployer audit log changed during deployment' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_MUTATE_AUDIT_PATH FAKE_ADAPTER_MUTATE_AUDIT_TARGET +[[ -L "$deployed_current" && -e "$deployed_current" && -f "$deployed_current/policy.conf" && -f "$deployed_current/state.json" ]] || fail 'post-publication failure left deployed/current dangling' +cmp -s "$deployed_current/policy.conf" "$config" || fail 'published deployed policy does not match the deployed configuration' +rm "$root/var/log/ci-fleet-deployer/audit.log" +write_evidence staging example-staging +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_FORBID_REQUEST_PATH=$request +export CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG=$tmp/inhibitor.log +expect_success "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_FORBID_REQUEST_PATH CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG +[[ $(<"$tmp/inhibitor.log") == deploy ]] || fail 'deployment transaction was not enclosed by the shutdown inhibitor' +[[ ! -e "$request" && -f "$root/var/lib/ci-fleet-deployer/last-request.conf" ]] || fail 'completed request was not consumed atomically' +deployed_current=$root/var/lib/ci-fleet-deployer/deployed/current +[[ -L "$deployed_current" && -f "$deployed_current/policy.conf" && -f "$deployed_current/state.json" ]] || fail 'deployed policy/state pair was not published through one pointer' +{ printf '# semantic replay with different bytes\n'; tac "$root/var/lib/ci-fleet-deployer/last-request.conf"; } >"$request" +chmod 0600 "$request" +expect_failure 'deployment request was already completed' "$runtime" deploy >/dev/null +cp "$root/var/lib/ci-fleet-deployer/last-request.conf" "$tmp/request-a" +cp "$approval" "$tmp/approval-a" +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=approval-20260808-2').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:01:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +expect_success "$runtime" deploy >/dev/null +cp "$tmp/approval-a" "$approval"; chmod 0600 "$approval" +cp "$tmp/request-a" "$request"; chmod 0600 "$request" +expect_failure 'deployment request was already consumed' "$runtime" deploy >/dev/null +grep -Fq 'sha256:bbbbbbbb' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'audit log omitted the immutable digest' +if grep -Fq 'registry.example.invalid' "$root/var/log/ci-fleet-deployer/audit.log"; then fail 'audit log exposed a private-capable endpoint'; fi +for operation in health cleanup deploy; do grep -Fxq "$operation" "$FAKE_ADAPTER_LOG" || fail "runtime did not invoke adapter $operation"; done +: >"$root/var/lib/ci-fleet-deployer/drained" +chmod 0600 "$root/var/lib/ci-fleet-deployer/drained" +expect_failure 'deployer is drained' "$runtime" deploy >/dev/null +expect_failure 'deployer is drained' "$runtime" cleanup >/dev/null +rm "$root/var/lib/ci-fleet-deployer/drained" +ln -s "$tmp/missing-drain-target" "$root/var/lib/ci-fleet-deployer/drained" +expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" deploy >/dev/null +expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" cleanup >/dev/null +rm "$root/var/lib/ci-fleet-deployer/drained" + +# Adapter upgrades use immutable versioned paths so the deployed policy keeps +# real rollback bytes while the candidate is validated and activated. +old_adapter=$adapter +old_adapter_sha=$(sha256sum "$old_adapter" | cut -d' ' -f1) +adapter=$root/etc/ci-fleet-deployer/adapters/application-adapter.v2 +cp "$old_adapter" "$adapter" +# shellcheck disable=SC2016 # Write literal adapter variables into the fixture. +printf '\nprintf "adapter-v2:%%s\\n" "$1" >>"${FAKE_ADAPTER_LOG:?}"\n' >>"$adapter" +chmod 0700 "$adapter" +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +grep -Fxq "ADAPTER_PATH=$old_adapter" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" || fail 'adapter upgrade did not retain the old adapter path for rollback' +grep -Fxq "ADAPTER_SHA256=$old_adapter_sha" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" || fail 'adapter upgrade did not retain the old adapter digest for rollback' +check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=yes' <<<"$check" || fail 'versioned adapter upgrade is not rollback-capable' + +# Recursive adapter cleanup of the writable state root must not destroy the +# retained rollback pair: its backup lives only in unexported process memory. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=state-root-delete-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:07:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +lkg_state_before=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +lkg_policy_before=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf") +last_request_before=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-request.conf") +printf 'deploy\n' >"$tmp/fail-after-state-delete" +export FAKE_ADAPTER_DELETE_STATE_ROOT=$root/var/lib/ci-fleet-deployer FAKE_ADAPTER_FAIL_AFTER_MUTATION=$tmp/fail-after-state-delete +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_DELETE_STATE_ROOT FAKE_ADAPTER_FAIL_AFTER_MUTATION +[[ $lkg_state_before == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'recursive state cleanup destroyed retained rollback state' +[[ $lkg_policy_before == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf")" ]] || fail 'recursive state cleanup destroyed retained rollback policy' +[[ $last_request_before == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-request.conf")" ]] || fail 'recursive state cleanup destroyed completed-deployment rollback marker' +compgen -G "$root/var/lib/ci-fleet-deployer/.lkg.*" >/dev/null && fail 'rollback backup remained inside adapter-writable state' +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'rollback after recursive state cleanup did not complete' + +# Bytes substituted into the live checkout after review must never reach a staged release. +cp "$runtime" "$tmp/runtime.saved" +restore_live_checkout() { + [[ ! -f "$tmp/runtime.saved" ]] || cat "$tmp/runtime.saved" >"$runtime" + git -C "$repo_root" replace -d "$core_ref" 2>/dev/null || true +} +trap 'restore_live_checkout; rm -rf "$tmp"' EXIT +release_before=$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh") +printf '# substituted-live-bytes\n' >>"$runtime" +expect_failure 'differs from the reviewed commit' "$installer" --repair --config "$config" >/dev/null +[[ $release_before == "$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh")" ]] || fail 'mutated live checkout bytes entered the trusted release' +replacement=$(git -C "$repo_root" commit-tree 'HEAD^{tree}' -m replace-fixture 2>/dev/null || true) +if [[ -n $replacement ]]; then + git -C "$repo_root" replace "$core_ref" "$replacement" + expect_failure 'differs from the reviewed commit' "$installer" --repair --config "$config" >/dev/null + [[ $release_before == "$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh")" ]] || fail 'replacement ref bytes entered the trusted release' + git -C "$repo_root" replace -d "$core_ref" +fi +cat "$tmp/runtime.saved" >"$runtime" +git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$runtime" || fail 'live checkout restoration diverged from HEAD' +rm -f "$root/var/lib/ci-fleet-deployer/drained" + +grep -Fq 'DEPLOYER-HOST.md' "$repo_root/docs/README.md" || fail 'operator index does not link the deployer runbook' +[[ -x "$repo_root/scripts/test-deployer-units.sh" ]] || fail 'real systemd unit verification is not wired' +grep -Fq 'scripts/test-deployer-units.sh' "$repo_root/scripts/validate.sh" || fail 'repository validation omits systemd unit verification' +lock_line=$(grep -n 'flock -n 9' "$runtime" | cut -d: -f1) +policy_line=$(grep -n "secure_file \"\$config\" 'deployer configuration'" "$runtime" | cut -d: -f1) +((lock_line < policy_line)) || fail 'runtime loads active policy before acquiring the shared operation lock' +for phrase in '--check' '--install' '--upgrade' '--repair' '--drain' '--resume' '--rollback' '--uninstall' 'manual-exact-head' 'github-environment' 'GitHub Free' 'PRODUCTION_AUTHORIZATION_EVIDENCE_PATH' 'CI_FLEET_DEPLOYER_ROLLBACK_COMMIT' 'application-owned' 'REPORT action='; do + grep -Fq -- "$phrase" "$repo_root/docs/DEPLOYER-HOST.md" || fail "deployer runbook omits $phrase" +done +for unit in "$repo_root"/deploy/deployer/*; do + grep -Fq 'ci-fleet-deployer' "$unit" || fail "unit is not deployer-scoped: $unit" +done + +printf 'DEPLOYER_INSTALL_TESTS_OK\n' diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 445a77d..ab5fd31 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -323,6 +323,13 @@ expect_failure 'host configuration must be owned by root' "$installer" --install unset FAKE_WRONG_HOST_CONFIG_OWNER expect_failure 'managed installs require the default' "$installer" --check "${base_args[@]}" --ref "$ref_one" --host-config "$tmp/custom-host.env" +# A controller must never install onto a deployer host. +mkdir -p "$root/etc/systemd/system" "$root/var/lib/ci-fleet-deployer" +printf '[Unit]\n' >"$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'deployer host state is present' "$installer" --install "${base_args[@]}" --ref "$ref_one" +rm "$root/etc/systemd/system/ci-fleet-deployer.service" +rmdir "$root/var/lib/ci-fleet-deployer" + first=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not converge' [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' diff --git a/scripts/validate.sh b/scripts/validate.sh index 9660aaa..e2b86cf 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -28,6 +28,8 @@ python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.jso python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null scripts/test-capacity-preflight.sh scripts/test-install-worker-controller.sh +scripts/test-install-deployer.sh +scripts/test-deployer-units.sh scripts/test-install-status-receiver.sh tmp=$(mktemp)