diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 006f5cf440..85aa56b885 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -272,9 +272,86 @@ jobs: - name: Log in to GHCR with Podman run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin + - name: Capture live rootless pasta state + run: | + set -euo pipefail + workdir="$(mktemp -d "${RUNNER_TEMP}/openshell-pasta-live.XXXXXX")" + socket="${workdir}/podman/podman.sock" + service_log="${workdir}/podman-service.log" + container_name="openshell-pasta-live-$$" + service_pid="" + cleanup() { + if [ -S "${socket}" ]; then + podman --url "unix://${socket}" rm --force "${container_name}" >/dev/null 2>&1 || true + fi + if [ -n "${service_pid}" ] && kill -0 "${service_pid}" 2>/dev/null; then + kill "${service_pid}" 2>/dev/null || true + wait "${service_pid}" 2>/dev/null || true + fi + } + trap cleanup EXIT + + mkdir -p "$(dirname "${socket}")" + podman system service --time=0 "unix://${socket}" >"${service_log}" 2>&1 & + service_pid="$!" + for _ in $(seq 1 30); do + if [ -S "${socket}" ] && podman --url "unix://${socket}" info >/dev/null 2>&1; then + break + fi + if ! kill -0 "${service_pid}" 2>/dev/null; then + cat "${service_log}" >&2 || true + exit 1 + fi + sleep 1 + done + podman --url "unix://${socket}" info >/dev/null + podman --url "unix://${socket}" run --detach --name "${container_name}" \ + docker.io/library/alpine:3.22 sleep infinity >/dev/null + + # Ubuntu selects pasta.avx2 on x86 hosts with AVX2 and pasta on + # other architectures. Match the executable path rather than the + # short process name so this capture covers both variants. + pasta_pids() { + pgrep -f '^/usr/bin/pasta(\.avx2)?( |$)' || true + } + echo "=== live rootless pasta state ===" + ps -eo pid,ppid,user,comm,args | grep -E '[p]odman|[p]asta' || true + for pid in "${service_pid}" $(pasta_pids); do + [ -r "/proc/${pid}/attr/current" ] || continue + echo "=== pid=${pid} ===" + sudo cat "/proc/${pid}/attr/current" || true + readlink "/proc/${pid}/ns/user" || true + readlink "/proc/${pid}/ns/net" || true + sudo cat "/proc/${pid}/cgroup" || true + readlink "/proc/${pid}/exe" || true + sudo cat "/proc/${pid}/cmdline" | tr '\0' ' ' || true + echo + done + - name: Run Podman E2E run: ${{ matrix.cmd }} + - name: Diagnose rootless Podman AppArmor labels + if: always() + run: | + set -x + dpkg-query -W -f='${Package} ${Version}\n' \ + apparmor apparmor-utils passt podman conmon + dpkg-query -L passt + sudo sed -n '1,240p' /etc/apparmor.d/usr.bin.pasta || true + sudo aa-status --profiled || true + sudo cat /sys/kernel/security/apparmor/profiles || true + ps -eo pid,ppid,user,comm,args | grep -E '[p]odman|[p]asta' || true + pasta_pids() { + pgrep -f '^/usr/bin/pasta(\.avx2)?( |$)' || true + } + for pid in $(pasta_pids) $(pgrep -x podman || true); do + echo "=== pid=${pid} ===" + sudo cat "/proc/${pid}/attr/current" || true + sudo tr '\0' ' ' < "/proc/${pid}/cmdline" || true + echo + done + - name: Print AppArmor denials if: always() run: sudo dmesg | grep -E 'apparmor=.*DENIED|profile="unprivileged_userns"' | tail -100 || true diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index ae28452056..b6b0cda663 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -28,12 +28,16 @@ nix/test-guest/ ├── cache-seal.sh ├── distros/ │ ├── ubuntu.nix +│ ├── ubuntu-26.04.nix │ ├── centos.nix │ ├── fedora.nix │ └── rocky.nix └── configuration/ ├── docker.yml ├── podman.yml + ├── podman-rootless-pasta.yml + ├── pasta-apparmor-control.yml + ├── pasta-apparmor-fix.yml └── selinux.yml ``` @@ -53,6 +57,7 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-gues | Distro | Docker | Podman | SELinux | Package format | | --- | --- | --- | --- | --- | | Ubuntu 24.04 | Yes | Yes | No | `.deb` | +| Ubuntu 26.04 | No | Rootless pasta + AppArmor | Yes | `.deb` | | CentOS Stream 10 | No | Yes | Yes | `.rpm` | | Fedora 44 | No | Yes | Yes | `.rpm` | | Rocky Linux 9 | Yes | Yes | Yes | `.rpm` | @@ -62,6 +67,53 @@ checks, but its Podman 4 release does not provide the `pasta` rootless network helper required by OpenShell sandbox callbacks. OpenShell Podman E2E runs use the Fedora guest, which provides Podman 5 and `pasta`. +## Rootless pasta AppArmor probe + +The `ubuntu-26.04` guest and `podman-rootless-pasta` configuration reproduce +the host-side AppArmor path that rootless Podman uses to stop `pasta`. The +configuration pins the same Podman 5.7.0 and conmon packages as the +`ubuntu-26.04` GitHub-hosted rootless E2E job, installs `passt` and the +rootless prerequisites, enables AppArmor and +`kernel.apparmor_restrict_unprivileged_userns=1`, then verifies +`true:pasta:/usr/bin/conmon`. The probe starts a private `podman system service` +with the same `/usr/bin/conmon` config override OpenShell E2E uses, rather than +using a systemd user socket. Because Ubuntu's package can advance outside this +repository, `pasta-apparmor-control` removes the upstream Podman signal +allowance from that packaged profile to create a deterministic pre-fix control. + +Copy the probe into an unmodified-package guest to test for the known denial: + +```shell +nix run .#test-guest -- \ + --distro ubuntu-26.04 \ + --with podman-rootless-pasta \ + --with pasta-apparmor-control \ + --copy "$PWD/nix/test-guest/pasta-signal-probe.sh:/usr/local/bin/pasta-signal-probe" \ + -- pasta-signal-probe --expect-denial +``` + +Apply `pasta-apparmor-fix` after the control configuration to add the upstream +rule `signal (receive) peer=podman,`, reload the profile, and verify the clean +path: + +```shell +nix run .#test-guest -- \ + --distro ubuntu-26.04 \ + --with podman-rootless-pasta \ + --with pasta-apparmor-control \ + --with pasta-apparmor-fix \ + --copy "$PWD/nix/test-guest/pasta-signal-probe.sh:/usr/local/bin/pasta-signal-probe" \ + -- pasta-signal-probe --expect-clean +``` + +Use `--keep` when either run fails to retain the serial log and writable guest +overlay for inspection. The probe tests one exact AppArmor signal denial; it +does not replace the rootless Podman OpenShell E2E suite. + +For a non-asserting collection run, use `pasta-signal-probe --report`. It emits +the versions, live Podman/pasta processes, AppArmor labels, and user/network +namespace identities while the pasta helper is running. + List the available distros and configurations: ```shell diff --git a/nix/test-guest/cache.sh b/nix/test-guest/cache.sh index 1103b3c289..c1823f0c6d 100644 --- a/nix/test-guest/cache.sh +++ b/nix/test-guest/cache.sh @@ -102,7 +102,7 @@ if [ -z "${distro}" ]; then usage >&2 exit 2 fi -if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]] || +if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9.-]*$ ]] || [ ! -r "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}" ]; then echo "unknown distro: ${distro}" >&2 exit 2 diff --git a/nix/test-guest/configuration/pasta-apparmor-control.yml b/nix/test-guest/configuration/pasta-apparmor-control.yml new file mode 100644 index 0000000000..f8c491c93c --- /dev/null +++ b/nix/test-guest/configuration/pasta-apparmor-control.yml @@ -0,0 +1,35 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Produce a deterministic pre-fix control from the Ubuntu-packaged profile. +# Ubuntu's package may advance independently of this repository, so relying on +# its version to retain the missing rule would make the regression probe flaky. + +- name: Remove the pasta AppArmor signal fix + hosts: test_vm + become: true + gather_facts: false + + tasks: + - name: Require the packaged pasta AppArmor profile + ansible.builtin.stat: + path: /etc/apparmor.d/usr.bin.pasta + register: pasta_profile + + - name: Reject guests without the pasta profile + ansible.builtin.assert: + that: + - pasta_profile.stat.exists + fail_msg: Apply podman-rootless-pasta before pasta-apparmor-control. + + - name: Remove the Podman signal allowance + ansible.builtin.lineinfile: + path: /etc/apparmor.d/usr.bin.pasta + regexp: '^ signal \(receive\) peer=podman,$' + state: absent + + - name: Reload the control pasta AppArmor profile + ansible.builtin.command: + cmd: apparmor_parser --replace /etc/apparmor.d/usr.bin.pasta + changed_when: true diff --git a/nix/test-guest/configuration/pasta-apparmor-fix.yml b/nix/test-guest/configuration/pasta-apparmor-fix.yml new file mode 100644 index 0000000000..337aaa7a07 --- /dev/null +++ b/nix/test-guest/configuration/pasta-apparmor-fix.yml @@ -0,0 +1,36 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Apply the upstream pasta AppArmor signal rule after podman-rootless-pasta has +# installed the packaged profile. This is deliberately an opt-in overlay so a +# guest using only podman-rootless-pasta remains an unmodified-package control. + +- name: Apply the pasta AppArmor signal fix + hosts: test_vm + become: true + gather_facts: false + + tasks: + - name: Require the packaged pasta AppArmor profile + ansible.builtin.stat: + path: /etc/apparmor.d/usr.bin.pasta + register: pasta_profile + + - name: Reject guests without the pasta profile + ansible.builtin.assert: + that: + - pasta_profile.stat.exists + fail_msg: Apply podman-rootless-pasta before pasta-apparmor-fix. + + - name: Allow pasta to receive signals from Podman + ansible.builtin.lineinfile: + path: /etc/apparmor.d/usr.bin.pasta + insertafter: '^ include $' + line: ' signal (receive) peer=podman,' + regexp: '^ signal \(receive\) peer=podman,$' + + - name: Reload the fixed pasta AppArmor profile + ansible.builtin.command: + cmd: apparmor_parser --replace /etc/apparmor.d/usr.bin.pasta + changed_when: true diff --git a/nix/test-guest/configuration/podman-rootless-pasta.yml b/nix/test-guest/configuration/podman-rootless-pasta.yml new file mode 100644 index 0000000000..04015ad483 --- /dev/null +++ b/nix/test-guest/configuration/podman-rootless-pasta.yml @@ -0,0 +1,115 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Configure the Ubuntu 26.04 rootless Podman/pasta environment used by the +# AppArmor regression probe. Keep this distinct from podman.yml: Ubuntu 24.04 +# intentionally remains a supported generic Podman guest even though it lacks +# the pasta helper required for OpenShell's rootless callback topology. + +- name: Configure rootless Podman with pasta and AppArmor + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate Ubuntu 26.04 support + ansible.builtin.assert: + that: + - ansible_facts.distribution == "Ubuntu" + - ansible_facts.distribution_version == "26.04" + fail_msg: >- + podman-rootless-pasta requires the Ubuntu 26.04 guest, not + {{ ansible_facts.distribution }} {{ ansible_facts.distribution_version }}. + + - name: Refresh Ubuntu package metadata + ansible.builtin.apt: + update_cache: true + + - name: Install rootless Podman and AppArmor dependencies + ansible.builtin.apt: + name: + - apparmor + - apparmor-utils + - conmon=2.1.13+ds1-2 + - fuse-overlayfs + - passt + # Match the Ubuntu 26.04 GitHub-hosted rootless E2E job. The + # surrounding Ubuntu package set is intentionally left to the + # selected cloud image; these are the two versions the workflow + # explicitly supports and verifies. + - podman=5.7.0+ds2-3build1 + - uidmap + state: present + + - name: Configure subordinate user and group ranges + ansible.builtin.lineinfile: + path: "{{ item.path }}" + line: "{{ ansible_facts.user_id }}:100000:65536" + regexp: "^{{ ansible_facts.user_id | regex_escape }}:" + create: true + owner: root + group: root + mode: "0644" + loop: + - path: /etc/subuid + - path: /etc/subgid + + - name: Enable AppArmor user namespace restrictions + ansible.builtin.copy: + dest: /etc/sysctl.d/60-openshell-pasta-apparmor.conf + owner: root + group: root + mode: "0644" + content: | + kernel.apparmor_restrict_unprivileged_userns = 1 + + - name: Apply AppArmor user namespace restrictions + ansible.builtin.command: + cmd: sysctl --system + changed_when: true + + - name: Start AppArmor + ansible.builtin.systemd_service: + name: apparmor.service + enabled: true + state: started + + - name: Verify AppArmor is enabled + ansible.builtin.command: + cmd: aa-status --enabled + changed_when: false + + - name: Verify pasta AppArmor profile is installed + ansible.builtin.command: + cmd: dpkg-query -L passt + register: passt_files + changed_when: false + + - name: Require the pasta AppArmor profile + ansible.builtin.assert: + that: + - "'/etc/apparmor.d/usr.bin.pasta' in passt_files.stdout_lines" + fail_msg: Ubuntu passt package did not install /etc/apparmor.d/usr.bin.pasta. + + - name: Reload the packaged pasta AppArmor profile + ansible.builtin.command: + cmd: apparmor_parser --replace /etc/apparmor.d/usr.bin.pasta + changed_when: true + + - name: Verify the rootless pasta environment + ansible.builtin.command: + cmd: >- + podman info --format + '{{ "{{" }}.Host.Security.Rootless{{ "}}" }}:{{ "{{" }}.Host.RootlessNetworkCmd{{ "}}" }}:{{ "{{" }}.Host.Conmon.Path{{ "}}" }}' + become: false + register: podman_environment + changed_when: false + + - name: Require rootless Podman with pasta + ansible.builtin.assert: + that: + - podman_environment.stdout == "true:pasta:/usr/bin/conmon" + fail_msg: >- + Expected rootless Podman with pasta and /usr/bin/conmon, got + {{ podman_environment.stdout | quote }}. diff --git a/nix/test-guest/default.nix b/nix/test-guest/default.nix index 2cfc772279..f31d2d2b0c 100644 --- a/nix/test-guest/default.nix +++ b/nix/test-guest/default.nix @@ -15,6 +15,7 @@ let distros = { ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; + "ubuntu-26.04" = import ./distros/ubuntu-26.04.nix { inherit pkgs architecture; }; centos = import ./distros/centos.nix { inherit pkgs architecture; }; fedora = import ./distros/fedora.nix { inherit pkgs architecture; }; rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; @@ -23,6 +24,9 @@ let configurations = { docker = ./configuration/docker.yml; podman = ./configuration/podman.yml; + podman-rootless-pasta = ./configuration/podman-rootless-pasta.yml; + pasta-apparmor-control = ./configuration/pasta-apparmor-control.yml; + pasta-apparmor-fix = ./configuration/pasta-apparmor-fix.yml; selinux = ./configuration/selinux.yml; }; diff --git a/nix/test-guest/distros/ubuntu-26.04.nix b/nix/test-guest/distros/ubuntu-26.04.nix new file mode 100644 index 0000000000..e14c970aa3 --- /dev/null +++ b/nix/test-guest/distros/ubuntu-26.04.nix @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +let + imageArchitecture = if architecture == "aarch64" then "arm64" else "amd64"; + imageUrl = "https://cloud-images.ubuntu.com/releases/server/26.04/release-20260717/ubuntu-26.04-server-cloudimg-${imageArchitecture}.img"; + imageHash = + if architecture == "aarch64" then + "sha256-WS/EdHgWo4EkG3nQSkVFqNp711d57XaQ10dj1jCkbfQ=" + else + "sha256-t9qg/8sGrPRFR6L3aCCI0TrPBgmTdQf6qNfvAwJkDd0="; +in +{ + osId = "ubuntu"; + osVersion = "26.04"; + packageFamily = "deb"; + inherit imageUrl imageHash; + image = pkgs.fetchurl { + name = "ubuntu-26.04-server-cloudimg-${imageArchitecture}.img"; + url = imageUrl; + hash = imageHash; + }; +} diff --git a/nix/test-guest/pasta-signal-probe.sh b/nix/test-guest/pasta-signal-probe.sh new file mode 100644 index 0000000000..e7e1c3af86 --- /dev/null +++ b/nix/test-guest/pasta-signal-probe.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Exercise the rootless Podman -> pasta SIGTERM path and assert either the +# packaged-profile control or the upstream-profile fix. + +set -Eeuo pipefail + +usage() { + echo "Usage: pasta-signal-probe.sh --expect-denial|--expect-clean|--report" >&2 +} + +if [ "$#" -ne 1 ]; then + usage + exit 2 +fi + +case "$1" in +--expect-denial) expected_denial=1 ;; +--expect-clean) expected_denial=0 ;; +--report) expected_denial="" ;; +*) + usage + exit 2 + ;; +esac + +if [ "$(podman info --format '{{.Host.Security.Rootless}}:{{.Host.RootlessNetworkCmd}}')" != "true:pasta" ]; then + echo "expected rootless Podman with pasta" >&2 + exit 1 +fi + +container_name="openshell-pasta-signal-probe-$$" +work_dir="$(mktemp -d)" +podman_socket="${work_dir}/podman/podman.sock" +podman_config="${work_dir}/containers.conf" +podman_service_log="${work_dir}/podman-service.log" +podman_service_pid="" + +printf '%s\n' \ + '[engine]' \ + 'conmon_path = ["/usr/bin/conmon"]' \ + > "${podman_config}" +export CONTAINERS_CONF_OVERRIDE="${podman_config}" + +cleanup_service() { + if [ -n "${podman_service_pid}" ] && kill -0 "${podman_service_pid}" 2>/dev/null; then + kill "${podman_service_pid}" 2>/dev/null || true + wait "${podman_service_pid}" 2>/dev/null || true + fi +} +cleanup() { + if [ -S "${podman_socket}" ]; then + podman --url "unix://${podman_socket}" rm --force "${container_name}" >/dev/null 2>&1 || true + fi + cleanup_service +} +trap cleanup EXIT + +mkdir -p "$(dirname "${podman_socket}")" +podman system service --time=0 "unix://${podman_socket}" >"${podman_service_log}" 2>&1 & +podman_service_pid=$! + +for _ in $(seq 1 30); do + if [ -S "${podman_socket}" ] && podman --url "unix://${podman_socket}" info >/dev/null 2>&1; then + break + fi + if ! kill -0 "${podman_service_pid}" 2>/dev/null; then + cat "${podman_service_log}" >&2 || true + echo "rootless Podman API service exited before becoming reachable" >&2 + exit 1 + fi + sleep 1 +done + +if ! podman --url "unix://${podman_socket}" info >/dev/null 2>&1; then + cat "${podman_service_log}" >&2 || true + echo "rootless Podman API service did not become reachable within 30 seconds" >&2 + exit 1 +fi + +log_process_state() { + echo '=== runner-match process state ===' + printf 'podman package: ' + dpkg-query -W -f='${Version}\n' podman + printf 'conmon package: ' + dpkg-query -W -f='${Version}\n' conmon + printf 'passt package: ' + dpkg-query -W -f='${Version}\n' passt + printf 'Podman service PID: %s\n' "${podman_service_pid}" + ps -eo pid,ppid,user,comm,args | grep -E '[p]odman|[p]asta' || true + for pid in "${podman_service_pid}" $(pgrep -x pasta || true) $(pgrep -x podman || true); do + [ -r "/proc/${pid}/attr/current" ] || continue + echo "=== pid=${pid} ===" + cat "/proc/${pid}/attr/current" || true + readlink "/proc/${pid}/ns/user" || true + readlink "/proc/${pid}/ns/net" || true + done +} + +# The guest is disposable. Clearing the kernel ring buffer makes the assertion +# independent of unrelated AppArmor events emitted during cloud-init or package +# configuration. +sudo dmesg --clear +podman --url "unix://${podman_socket}" run --detach --name "${container_name}" docker.io/library/alpine:3.22 sleep infinity >/dev/null +log_process_state +started_at="$(date +%s)" +podman --url "unix://${podman_socket}" stop --time 15 "${container_name}" >/dev/null +elapsed_seconds="$(( $(date +%s) - started_at ))" + +new_dmesg="$(sudo dmesg)" +denials="$(printf '%s\n' "${new_dmesg}" | grep 'profile="pasta".*requested_mask="receive".*signal=term.*peer="podman"' || true)" + +if [ -z "${expected_denial}" ]; then + if [ -n "${denials}" ]; then + echo "observed pasta AppArmor denial after ${elapsed_seconds}s" + else + echo "no pasta AppArmor denial observed; stop completed in ${elapsed_seconds}s" + fi + exit 0 +fi + +if [ "${expected_denial}" -eq 1 ]; then + if [ -z "${denials}" ]; then + echo "expected a pasta SIGTERM receive denial, but none was emitted" >&2 + exit 1 + fi + if [ "${elapsed_seconds}" -lt 15 ]; then + echo "expected the SIGKILL fallback delay, but stop completed in ${elapsed_seconds}s" >&2 + exit 1 + fi + echo "observed expected pasta AppArmor denial after ${elapsed_seconds}s" + exit 0 +fi + +if [ -n "${denials}" ]; then + echo "pasta AppArmor still denied Podman's SIGTERM:" >&2 + printf '%s\n' "${denials}" >&2 + exit 1 +fi +if [ "${elapsed_seconds}" -ge 15 ]; then + echo "podman stop still reached the SIGKILL fallback delay (${elapsed_seconds}s)" >&2 + exit 1 +fi +echo "pasta accepted Podman's SIGTERM; stop completed in ${elapsed_seconds}s" diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index c999679492..f8a369624e 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -130,7 +130,7 @@ if [ -z "${distro}" ]; then usage >&2 exit 2 fi -if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]] || +if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9.-]*$ ]] || [ ! -r "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}" ]; then echo "unknown distro: ${distro}" >&2 exit 2