Merge branch 'master' into ci-incremental-caching

.github/workflows/build.yaml

@@ -51,14 +51,6 @@ jobs:
         key: kernel-ccache-${{ steps.kernel-submodule.outputs.ref }}
         restore-keys: kernel-ccache-
 
-    - name: Install dependencies
-      run: sudo apt-get update && sudo apt-get install -y bc img2simg
-
-    - name: Install python2
-      uses: gabrielfalcao/pyenv-action@2f49ca7587f9d0663d13f1147b78d3361417eaf7
-      with:
-        default: '2.7.18'
-
     - name: Build kernel
       run: ./build_kernel.sh
 

Dockerfile.kernel

@@ -0,0 +1,29 @@
+FROM ubuntu:20.04
+
+ARG UNAME
+ARG UID
+ARG GID
+
+# Set non-interactive installation
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Update and install necessary packages
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    python2 \
+    build-essential \
+    libssl-dev \
+    bc \
+    python-is-python2 \
+    openssl && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \
+    userdel -r `getent passwd ${UID} | cut -d : -f 1` > /dev/null 2>&1; \
+    groupdel -f `getent group ${GID} | cut -d : -f 1` > /dev/null 2>&1; \
+    groupadd -g ${GID} -o ${UNAME} && \
+    useradd -u $UID -g $GID ${UNAME} \
+;fi
+
+# Set entrypoint to keep container running
+ENTRYPOINT ["tail", "-f", "/dev/null"]

Dockerfile.sparsify

@@ -1,9 +1,20 @@
 FROM ubuntu:24.04
 
+ARG UNAME
+ARG UID
+ARG GID
+
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
     apt-get install -y android-sdk-libsparse-utils && \
     rm -rf /var/lib/apt/lists/*
 
+RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \
+      userdel -r `getent passwd ${UID} | cut -d : -f 1` > /dev/null 2>&1; \
+      groupdel -f `getent group ${GID} | cut -d : -f 1` > /dev/null 2>&1; \
+      groupadd -g ${GID} -o ${UNAME} && \
+      useradd -u $UID -g $GID ${UNAME}; \
+    fi
+
 ENTRYPOINT ["tail", "-f", "/dev/null"]

README.md

@@ -36,6 +36,20 @@ Validating changes:
 * [CI](https://github.com/commaai/agnos-builder/blob/master/.github/workflows/build.yaml) ensures the kernel and system builds work (and pushes the images for you to download)
 * [this](https://github.com/commaai/agnos-builder/blob/master/internal/README.md) is the checklist we go through before shipping new AGNOS releases to openpilot
 
+## Development - macOS
+
+Building the kernel on macOS requires the volume you work in to be [Case-sensitive](https://support.apple.com/lv-lv/guide/disk-utility/dsku19ed921c/mac).
+
+The recommended way is to create an APFS (Case-sensitive) volume following these [instructions](https://support.apple.com/lv-lv/guide/disk-utility/add-delete-or-erase-apfs-volumes-dskua9e6a110/22..6/mac/14.0) and checkout `agnos-builder` in that specific volume (or move it there). Not setting "Reserve Size" and "Quota Size" will make the volume dynamically allocate the space, which is also recommended.
+
+The same thing can be accomplished with the following command:
+```
+diskutil apfs addVolume disk3 "Case-sensitive APFS" agnos
+```
+* replace `disk3` with the `synthesized` disk in `diskutil list` that includes your main volume (e.g. Macintosh HD) - usually `disk3` on M-series Macs, `disk1` on Intel Macs
+* `agnos` is the name of the new volume - can be replaced with an arbitrary name
+* the volume is mounted automatically in `/Volumes/agnos`
+
 ## Contributing
 
 Join us in the `#dev-agnos` channel on [Discord](https://discord.comma.ai).

build_kernel.sh

@@ -1,4 +1,5 @@
-#!/bin/bash -e
+#!/bin/bash
+set -e
 
 DEFCONFIG=tici_defconfig
 
@@ -10,62 +11,92 @@ OUTPUT_DIR=$DIR/output
 BOOT_IMG=./boot.img
 cd $DIR
 
+if [[ "$(uname)" == 'Darwin' ]]; then
+  echo "---------------   macOS support   ---------------"
+  echo "Ensure you are in a Case-sensitive APFS volume to build the AGNOS kernel."
+  echo "https://github.com/commaai/agnos-builder?tab=readme-ov-file#development---macos"
+  echo "-------------------------------------------------"
+  echo "Press any key to continue or CTRL+C to abort..."
+  read -n 1 -s
+fi
+
+# Setup kernel build container
+docker build -f Dockerfile.kernel -t agnos-kernel $DIR \
+   --build-arg UNAME=$(id -nu) \
+   --build-arg UID=$(id -u) \
+   --build-arg GID=$(id -g)
+echo "Starting agnos-kernel container"
+CONTAINER_ID=$(docker run -d -v $DIR:$DIR -w $DIR agnos-kernel)
+
+# Cleanup container on exit
+trap "echo \"Cleaning up container:\"; \
+docker container rm -f $CONTAINER_ID" EXIT
+
 # Clone kernel if not done already
 if git submodule status --cached agnos-kernel-sdm845/ | grep "^-"; then
+  echo "Cloning agnos-kernel-sdm845"
   git submodule update --init agnos-kernel-sdm845
 fi
-cd agnos-kernel-sdm845
 
-# Build parameters
-ARCH=$(uname -m)
-if [ "$ARCH" != "arm64" ] && [ "$ARCH" != "aarch64" ]; then
-  export CROSS_COMPILE=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-
-  export CC=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-gcc
-  export LD=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-ld.bfd
+$DIR/tools/extract_tools.sh
 
-  $DIR/tools/extract_tools.sh
-fi
-export ARCH=arm64
-
-# these do anything?
-export KCFLAGS="-w"
-
-# Load defconfig and build kernel
-echo "-- First make --"
-make $DEFCONFIG O=out
-echo "-- Second make: $(nproc --all) cores --"
-make -j$(nproc --all) O=out  # Image.gz-dtb
-
-# Turn on if you want perf
-# LDFLAGS=-static make -j$(nproc --all) -C tools/perf
-
-# Copy over Image.gz-dtb
-mkdir -p $TMP_DIR
-cd $TMP_DIR
-cp $DIR/agnos-kernel-sdm845/out/arch/arm64/boot/Image.gz-dtb .
-
-# Make boot image
-$TOOLS/mkbootimg \
-  --kernel Image.gz-dtb \
-  --ramdisk /dev/null \
-  --cmdline "console=ttyMSM0,115200n8 quiet loglevel=3 earlycon=msm_geni_serial,0xA84000 androidboot.hardware=qcom androidboot.console=ttyMSM0 ehci-hcd.park=3 lpm_levels.sleep_disabled=1 service_locator.enable=1 androidboot.selinux=permissive firmware_class.path=/lib/firmware/updates net.ifnames=0 dyndbg=\"\"" \
-  --pagesize 4096 \
-  --base 0x80000000 \
-  --kernel_offset 0x8000 \
-  --ramdisk_offset 0x8000 \
-  --tags_offset 0x100 \
-  --output $BOOT_IMG.nonsecure
-
-# le signing
-openssl dgst -sha256 -binary $BOOT_IMG.nonsecure > $BOOT_IMG.sha256
-openssl pkeyutl -sign -in $BOOT_IMG.sha256 -inkey $DIR/vble-qti.key -out $BOOT_IMG.sig -pkeyopt digest:sha256 -pkeyopt rsa_padding_mode:pkcs1
-dd if=/dev/zero of=$BOOT_IMG.sig.padded bs=2048 count=1
-dd if=$BOOT_IMG.sig of=$BOOT_IMG.sig.padded conv=notrunc
-cat $BOOT_IMG.nonsecure $BOOT_IMG.sig.padded > $BOOT_IMG
-
-# Copy to output dir
-mkdir -p $OUTPUT_DIR
-mv $BOOT_IMG $OUTPUT_DIR/
-cp $DIR/agnos-kernel-sdm845/out/techpack/audio/asoc/snd-soc-sdm845.ko $OUTPUT_DIR/
-cp $DIR/agnos-kernel-sdm845/out/techpack/audio/asoc/codecs/snd-soc-wcd9xxx.ko $OUTPUT_DIR/
-cp $DIR/agnos-kernel-sdm845/out/drivers/staging/qcacld-3.0/wlan.ko $OUTPUT_DIR/
+build_kernel() {
+  cd agnos-kernel-sdm845
+
+  # Build parameters
+  ARCH=$(uname -m)
+  if [ "$ARCH" != "arm64" ] && [ "$ARCH" != "aarch64" ]; then
+    export CROSS_COMPILE=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-
+    export CC=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-gcc
+    export LD=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-ld.bfd
+  fi
+
+  # Build arm64 arch
+  export ARCH=arm64
+
+  # Disable all warnings
+  export KCFLAGS="-w"
+
+  # Load defconfig and build kernel
+  echo "-- First make --"
+  make $DEFCONFIG O=out
+  echo "-- Second make: $(nproc --all) cores --"
+  make -j$(nproc --all) O=out  # Image.gz-dtb
+
+  # Turn on if you want perf
+  # LDFLAGS=-static make -j$(nproc --all) -C tools/perf
+
+  # Copy over Image.gz-dtb
+  mkdir -p $TMP_DIR
+  cd $TMP_DIR
+  cp $DIR/agnos-kernel-sdm845/out/arch/arm64/boot/Image.gz-dtb .
+
+  # Make boot image
+  $TOOLS/mkbootimg \
+    --kernel Image.gz-dtb \
+    --ramdisk /dev/null \
+    --cmdline "console=ttyMSM0,115200n8 quiet loglevel=3 earlycon=msm_geni_serial,0xA84000 androidboot.hardware=qcom androidboot.console=ttyMSM0 ehci-hcd.park=3 lpm_levels.sleep_disabled=1 service_locator.enable=1 androidboot.selinux=permissive firmware_class.path=/lib/firmware/updates net.ifnames=0 dyndbg=\"\"" \
+    --pagesize 4096 \
+    --base 0x80000000 \
+    --kernel_offset 0x8000 \
+    --ramdisk_offset 0x8000 \
+    --tags_offset 0x100 \
+    --output $BOOT_IMG.nonsecure
+
+  # le signing
+  openssl dgst -sha256 -binary $BOOT_IMG.nonsecure > $BOOT_IMG.sha256
+  openssl pkeyutl -sign -in $BOOT_IMG.sha256 -inkey $DIR/vble-qti.key -out $BOOT_IMG.sig -pkeyopt digest:sha256 -pkeyopt rsa_padding_mode:pkcs1
+  dd if=/dev/zero of=$BOOT_IMG.sig.padded bs=2048 count=1
+  dd if=$BOOT_IMG.sig of=$BOOT_IMG.sig.padded conv=notrunc
+  cat $BOOT_IMG.nonsecure $BOOT_IMG.sig.padded > $BOOT_IMG
+
+  # Copy to output dir
+  mkdir -p $OUTPUT_DIR
+  mv $BOOT_IMG $OUTPUT_DIR/
+  cp $DIR/agnos-kernel-sdm845/out/techpack/audio/asoc/snd-soc-sdm845.ko $OUTPUT_DIR/
+  cp $DIR/agnos-kernel-sdm845/out/techpack/audio/asoc/codecs/snd-soc-wcd9xxx.ko $OUTPUT_DIR/
+  cp $DIR/agnos-kernel-sdm845/out/drivers/staging/qcacld-3.0/wlan.ko $OUTPUT_DIR/
+}
+
+# Run build_kernel in container
+docker exec -u $(id -nu) $CONTAINER_ID bash -c "set -e; export DEFCONFIG=$DEFCONFIG DIR=$DIR TOOLS=$TOOLS TMP_DIR=$TMP_DIR OUTPUT_DIR=$OUTPUT_DIR BOOT_IMG=$BOOT_IMG; $(declare -f build_kernel); build_kernel"

build_system.sh

@@ -50,7 +50,11 @@ CONTAINER_ID=$(docker container create --entrypoint /bin/bash agnos-builder:late
 
 # Setup mount container for macOS and CI support (namespace.so)
 echo "Building agnos-mount docker image"
-docker build -f Dockerfile.sparsify -t agnos-mount $DIR
+docker build -f Dockerfile.sparsify -t agnos-mount $DIR \
+  --build-arg UNAME=$(id -nu) \
+  --build-arg UID=$(id -u) \
+  --build-arg GID=$(id -g)
+
 echo "Starting agnos-mount container"
 MOUNT_CONTAINER_ID=$(docker run -d --privileged -v $DIR:$DIR agnos-mount)
 
@@ -60,17 +64,13 @@ docker container rm -f $CONTAINER_ID $MOUNT_CONTAINER_ID" EXIT
 
 # Define functions for docker execution
 exec_as_user() {
-  docker exec -u $USERNAME $MOUNT_CONTAINER_ID "$@"
+  docker exec -u $(id -nu) $MOUNT_CONTAINER_ID "$@"
 }
 
 exec_as_root() {
   docker exec $MOUNT_CONTAINER_ID "$@"
 }
 
-# Create host user in container (fixes namespace.so error)
-USERNAME=$(whoami)
-exec_as_root useradd --uid $(id -u) -U -m $USERNAME &> /dev/null
-
 # Create filesystem ext4 image
 echo "Creating empty filesystem"
 exec_as_user fallocate -l $ROOTFS_IMAGE_SIZE $ROOTFS_IMAGE

tools/extract_tools.sh

@@ -1,7 +1,5 @@
 #!/bin/bash -e
 
-echo "Extracting tools..."
-
 git lfs &> /dev/null || {
   echo "ERROR: git lfs not installed"
   exit 1
@@ -13,11 +11,6 @@ ROOT=$DIR/..
 
 cd $DIR
 
-LINARO_GCC=aarch64-linux-gnu-gcc
-GOOGLE_GCC_4_9=aarch64-linux-android-4.9
-EDK2_LLVM=llvm-arm-toolchain-ship
-SEC_IMAGE=SecImage
-
 # grep for `-`, which stands for LFS pointer
 git lfs ls-files | awk '{print $2}' | grep "-" &>/dev/null && {
   echo "Pulling git lfs objects..."
@@ -27,23 +20,32 @@ git lfs ls-files | awk '{print $2}' | grep "-" &>/dev/null && {
   cd $DIR
 }
 
+LINARO_GCC=aarch64-linux-gnu-gcc
+GOOGLE_GCC_4_9=aarch64-linux-android-4.9
+EDK2_LLVM=llvm-arm-toolchain-ship
+SEC_IMAGE=SecImage
+
 LINARO_GCC_TARBALL=$LINARO_GCC.tar.gz
 GOOGLE_GCC_4_9_TARBALL=$GOOGLE_GCC_4_9.tar.gz
 EDK2_LLVM_TARBALL=$EDK2_LLVM.tar.gz
 SEC_IMAGE_TARBALL=$SEC_IMAGE.tar.gz
 
-if [ ! -d $LINARO_GCC ]; then
-  tar -xzf $LINARO_GCC_TARBALL &>/dev/null
-fi
+ARCH=$(uname -m)
+if [ "$ARCH" != "arm64" ] && [ "$ARCH" != "aarch64" ]; then
+  echo "Extracting tools..."
+  if [ ! -d $LINARO_GCC ]; then
+    tar -xzf $LINARO_GCC_TARBALL &>/dev/null
+  fi
 
-if [ ! -d $GOOGLE_GCC_4_9 ]; then
-  tar -xzf $GOOGLE_GCC_4_9_TARBALL &>/dev/null
-fi
+  if [ ! -d $GOOGLE_GCC_4_9 ]; then
+    tar -xzf $GOOGLE_GCC_4_9_TARBALL &>/dev/null
+  fi
 
-if [ ! -d $EDK2_LLVM ]; then
-  tar -xzf $EDK2_LLVM_TARBALL &>/dev/null
-fi
+  if [ ! -d $EDK2_LLVM ]; then
+    tar -xzf $EDK2_LLVM_TARBALL &>/dev/null
+  fi
 
-if [ ! -d $SEC_IMAGE ]; then
-  tar -xzf $SEC_IMAGE_TARBALL &>/dev/null
+  if [ ! -d $SEC_IMAGE ]; then
+    tar -xzf $SEC_IMAGE_TARBALL &>/dev/null
+  fi
 fi

userspace/pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
   "evdev",
   "flake8",
   "flask",
+  "future-fstrings",  # needed for acados
   "gunicorn",
   "hatanaka == 2.4",
   "hexdump",
@@ -26,7 +27,7 @@ dependencies = [
   "libusb1",
   "markdown-it-py",
   "nose",
-  "numpy",
+  "numpy < 2.0.0",  # pending bumping in openpilot
   "onnx",
   "onnxruntime >= 1.16.3",
   "pillow",
@@ -39,7 +40,7 @@ dependencies = [
   "pydub",
   "pyjwt",
   "pylint",
-#"pyopencl == 2024.1", #FIXME: has some installation errors
+  "pyopencl == 2024.1",  # pinned until cmake update coming with 24.04
   "pyserial",
   "pytest",
   "pytest-cov",