diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index 937e774db7..41d4410d7d 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -204,6 +204,16 @@ jobs: openshell gateway select snap-docker openshell status + - name: Verify VM driver resolves at its layout path + run: | + set -euo pipefail + # Enters the snap's mount namespace: /usr/libexec/openshell only exists + # there, via the snapcraft layout: symlink into $SNAP. + OUTPUT="$(snap run --shell openshell.gateway -c \ + '/usr/libexec/openshell/openshell-driver-vm --version')" + echo "$OUTPUT" + grep -q '^openshell-driver-vm ' <<<"$OUTPUT" + kubernetes: name: Kubernetes Helm (kind) if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 768f5bd5a6..a38e8d83b0 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -661,7 +661,7 @@ jobs: build-snap: name: Build Snap - needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-supervisor-binary-linux] + needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-supervisor-binary-linux, build-driver-vm-linux] uses: ./.github/workflows/snap-package.yml with: checkout-ref: ${{ github.sha }} diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 35792bcb63..d04e8ea740 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -699,7 +699,7 @@ jobs: build-snap: name: Build Snap - needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-supervisor-binary-linux] + needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-supervisor-binary-linux, build-driver-vm-linux] uses: ./.github/workflows/snap-package.yml with: checkout-ref: ${{ inputs.tag || github.ref }} diff --git a/.github/workflows/snap-package.yml b/.github/workflows/snap-package.yml index ed248b79e5..ed6fc3a6e5 100644 --- a/.github/workflows/snap-package.yml +++ b/.github/workflows/snap-package.yml @@ -94,12 +94,18 @@ jobs: name: supervisor-binary-linux-${{ matrix.arch }} path: prebuilt/sandbox + - name: Download prebuilt VM driver binary + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: driver-vm-linux-${{ matrix.arch }} + path: prebuilt/driver-vm + - name: Extract prebuilt binaries run: | set -euo pipefail - mkdir -p prebuilt/{cli,gateway,sandbox} + mkdir -p prebuilt/{cli,gateway,sandbox,driver-vm} - for d in cli gateway sandbox; do + for d in cli gateway sandbox driver-vm; do for tarball in prebuilt/$d/*.tar.gz; do if [ -f "$tarball" ]; then tar -xzf "$tarball" -C "prebuilt/$d" @@ -118,6 +124,7 @@ jobs: cp prebuilt/cli/openshell snap/prebuilt/openshell cp prebuilt/gateway/openshell-gateway snap/prebuilt/openshell-gateway cp prebuilt/sandbox/openshell-sandbox snap/prebuilt/openshell-sandbox + cp prebuilt/driver-vm/openshell-driver-vm snap/prebuilt/openshell-driver-vm cp tasks/scripts/snap-gateway-wrapper.sh snap/prebuilt/openshell-gateway-wrapper cp LICENSE snap/prebuilt/ diff --git a/architecture/build.md b/architecture/build.md index 5c5751772a..6d3a216e15 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -138,10 +138,11 @@ Runtime layout: `GLIBC_2.28`; release workflows verify this before publishing artifacts. The gateway bundles z3, so the image does not need a distro-provided z3 runtime. - **VM driver**: host GNU-linked binary installed at - `/usr/libexec/openshell/openshell-driver-vm` in Linux packages and published - as a release artifact. Linux GNU VM driver binaries must not reference - `GLIBC_*` symbols newer than `GLIBC_2.28`; release workflows verify this - before publishing artifacts. + `/usr/libexec/openshell/openshell-driver-vm` in Linux packages (Debian, RPM) and at + `$SNAP/usr/libexec/openshell` in the snap (also exposed inside the snapped environment + via symlink at `/usr/libexec/openshell/openshe`), and published as a release artifact. + Linux GNU VM driver binaries must not reference `GLIBC_*` symbols newer than + `GLIBC_2.28`; release workflows verify this before publishing artifacts. - **Supervisor**: Alpine base with `nftables`, static binary at `/openshell-sandbox` (musl by default; see `SUPERVISOR_LIBC` above). Static linkage keeps the binary usable when the image is mounted/extracted into diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 19ac66c3f9..3066a306e6 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -285,6 +285,23 @@ in `post_install`, and owns the `brew services` gateway lifecycle. The service also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless the user explicitly overrides it. +On Linux, the OpenShell snap stages the prebuilt driver into `usr/libexec/openshell/` and a +`layout:` symlink exposes it at `/usr/libexec/openshell` inside snap confinement. Operators must +connect the `kvm` interface to grant the gateway access to `/dev/kvm`: + +```shell +sudo snap connect openshell:kvm +``` + +Without the `kvm` interface connected, the VM driver cannot create VMs. + +The snap also vendors `e2fsprogs` into `$SNAP/usr/sbin`. The driver shells out to +`mke2fs`/`mkfs.ext4` to format sandbox root disks and to `debugfs` to inject the supervisor and +guest init script. The `core24` base ships those binaries under `/usr/sbin`, but the snap +AppArmor profile does not permit executing base-snap `sbin` binaries: they resolve on `PATH` +and then fail `execve` with `EACCES`. The snap `PATH` prefers `$SNAP/usr/sbin`, so the vendored +copies win. + ## TODOs - The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` notes in `crates/openshell-server/src/lib.rs` and `crates/openshell-server/src/compute/vm.rs`. diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index e464d3da42..4558c9f39e 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -138,6 +138,24 @@ a snap refresh when you need the updated binary: sudo systemctl restart snap.openshell.gateway ``` +### VM compute driver + +The snap ships `openshell-driver-vm` at `/usr/libexec/openshell/openshell-driver-vm`, exposed +through a snapcraft `layout:` symlink. To enable it, set `compute_drivers = ["vm"]` in +`$SNAP_COMMON/gateway.toml`, then connect the `kvm` interface and restart the service: + +```shell +sudo snap connect openshell:kvm +sudo snap restart openshell.gateway +``` + +The `kvm` interface grants access to `/dev/kvm`, which the VM driver requires. Without it, the +gateway cannot create VMs even with the driver configured. + +VM driver state defaults to a path under the revision-scoped snap user data directory and does not +persist across snap refreshes. Set `[openshell.drivers.vm].state_dir` in `$SNAP_COMMON/gateway.toml` +to keep VM state in a stable location across refreshes. + ## Kubernetes Kubernetes deployments use the OpenShell Helm chart. For step-by-step installation, refer to [Kubernetes Setup](/kubernetes/setup). For chart values and packaging details, refer to the [Helm chart README](https://github.com/NVIDIA/OpenShell/blob/main/deploy/helm/openshell/README.md). diff --git a/snapcraft.yaml b/snapcraft.yaml index d221c51e7e..1128420870 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -35,6 +35,11 @@ description: | openshell status openshell gateway add http://127.0.0.1:17670 --local --name openshell-gateway + To use the VM compute driver instead of Docker, connect the kvm + interface and select the driver in $SNAP_COMMON/gateway.toml: + + sudo snap connect openshell:kvm + After a snap refresh, restart the gateway to pick up the new revision: sudo snap restart openshell.gateway @@ -58,6 +63,13 @@ platforms: build-on: [arm64] build-for: [arm64] +layout: + # The gateway resolves openshell-driver-vm through its conventional libexec + # search paths. Symlink /usr/libexec/openshell at the snap-internal copy so + # that resolution works inside the snap mount namespace. + /usr/libexec/openshell: + symlink: $SNAP/usr/libexec/openshell + apps: openshell: command: bin/openshell @@ -98,6 +110,9 @@ apps: - docker # Docker snap is required because the snap uses the docker:docker-daemon # interface slot. It does not work with system-installed Docker. + # kvm is required by the VM compute driver for /dev/kvm access. The + # interface must be connected manually: sudo snap connect openshell:kvm + - kvm - log-observe - network - network-bind @@ -114,7 +129,7 @@ parts: set -euo pipefail MISSING=() - for bin in openshell openshell-gateway openshell-sandbox openshell-gateway-wrapper; do + for bin in openshell openshell-gateway openshell-sandbox openshell-driver-vm openshell-gateway-wrapper; do if [ ! -f "$CRAFT_PART_SRC/$bin" ]; then MISSING+=("$bin") fi @@ -136,6 +151,8 @@ parts: "$CRAFT_PART_INSTALL/bin/openshell-gateway" install -D -m 0755 "$CRAFT_PART_SRC/openshell-sandbox" \ "$CRAFT_PART_INSTALL/bin/openshell-sandbox" + install -D -m 0755 "$CRAFT_PART_SRC/openshell-driver-vm" \ + "$CRAFT_PART_INSTALL/usr/libexec/openshell/openshell-driver-vm" install -D -m 0755 "$CRAFT_PART_SRC/openshell-gateway-wrapper" \ "$CRAFT_PART_INSTALL/bin/openshell-gateway-wrapper" install -D -m 0644 "$CRAFT_PART_SRC/meta/gui/term.desktop" \ @@ -157,3 +174,15 @@ parts: - openssh-client prime: - usr/bin/ssh + + e2fsprogs: + # The VM compute driver shells out to mke2fs/mkfs.ext4 to format sandbox + # root disks and to debugfs to inject files into them. core24 ships these + # under /usr/sbin, but the snap AppArmor profile does not permit executing + # base-snap sbin binaries — they resolve on PATH and then fail execve with + # EACCES. Vendor them into $SNAP, which the snap PATH prefers over + # /usr/sbin. The shared libraries are staged alongside because the base's + # copies are not guaranteed to match the vendored binaries. + plugin: nil + stage-packages: + - e2fsprogs diff --git a/tasks/ci.toml b/tasks/ci.toml index 4c0b5f8ea7..83a0eaf24a 100644 --- a/tasks/ci.toml +++ b/tasks/ci.toml @@ -27,12 +27,33 @@ description = "Build all Rust crates in release mode" run = "cargo build --workspace --release" hide = true +# The VM driver embeds its runtime from OPENSHELL_VM_RUNTIME_COMPRESSED_DIR, which has no +# default in crates/openshell-driver-vm/build.rs. When the directory is missing or incomplete, +# build.rs still exits 0 but embeds empty stub resources, and the driver only fails later at +# sandbox creation with "VM runtime not embedded" — so guard it here instead. +# +# Use $MISE_PROJECT_ROOT, not {{config_root}}: this file is pulled in through +# [task_config] includes, so {{config_root}} renders as /tasks. ["build:rust:snap"] description = "Build release Rust binaries consumed by the hand-staged snap" run = [ + """ + runtime_dir="$MISE_PROJECT_ROOT/target/vm-runtime-compressed" + for f in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst openshell-sandbox.zst; do + if [ ! -f "$runtime_dir/$f" ]; then + echo "ERROR: missing $runtime_dir/$f" >&2 + echo "Run 'mise run vm:setup && mise run vm:supervisor' before building the snap." >&2 + exit 1 + fi + done + """, "cargo build --release -p openshell-cli", "cargo build --release -p openshell-server --features bundled-z3", "cargo build --release -p openshell-sandbox", + """ + OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="$MISE_PROJECT_ROOT/target/vm-runtime-compressed" \ + cargo build --release -p openshell-driver-vm + """, ] [check]