From 0262fe77c428cd93390b098929894bd7d2f81c3c Mon Sep 17 00:00:00 2001 From: Andrei Radulescu Date: Fri, 26 Jul 2024 20:13:39 +0300 Subject: [PATCH 1/5] fix typo in README (#266) fixed typo in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f2450fba..fef7fb17 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # agnos-builder -This is the buidler repo for AGNOS, our Ubuntu based OS. AGNOS runs on the comma three devkit and [comma 3X](https://comma.ai/shop/comma-3x). +This is the builder repo for AGNOS, our Ubuntu based OS. AGNOS runs on the comma three devkit and [comma 3X](https://comma.ai/shop/comma-3x). NOTE: the `edk2_tici` and `agnos-firmware` submodules are internal, private repos. @@ -71,6 +71,6 @@ Some concrete things on the roadmap: - [ ] make the image tiny, for fast updating and flashing https://github.com/commaai/agnos-builder/issues/225 - [ ] update to Ubuntu 24.04 https://github.com/commaai/openpilot/issues/32386 - [ ] mainline Linux kernel https://github.com/commaai/openpilot/issues/32386 -- [ ] fully open source +- [ ] fully open source - [ ] anything from `agnos-firmware`: XBL, ABL, etc. - [ ] open source Weston https://github.com/commaai/agnos-builder/issues/16 From f2498d1f4ba1e959c8d48733bd90aab402587f2a Mon Sep 17 00:00:00 2001 From: martinl Date: Sun, 28 Jul 2024 20:27:20 +0300 Subject: [PATCH 2/5] Update README.md (#269) Add note about flash.comma.ai --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fef7fb17..a1989f10 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ This is the builder repo for AGNOS, our Ubuntu based OS. AGNOS runs on the comma three devkit and [comma 3X](https://comma.ai/shop/comma-3x). -NOTE: the `edk2_tici` and `agnos-firmware` submodules are internal, private repos. +NOTE #1: the `edk2_tici` and `agnos-firmware` submodules are internal, private repos. + +NOTE #2: Use https://flash.comma.ai if you just want to reflash your device with latest AGNOS version ## Development From 54d68a5df9ae06d5656a66bfeae73d08215ad2a9 Mon Sep 17 00:00:00 2001 From: Andrei Radulescu Date: Sun, 28 Jul 2024 20:41:09 +0300 Subject: [PATCH 3/5] support cursor server (vscode fork) (#267) support for cursor server (vscode fork) --- userspace/usr/comma/comma.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userspace/usr/comma/comma.sh b/userspace/usr/comma/comma.sh index 8f7414d5..031b4406 100755 --- a/userspace/usr/comma/comma.sh +++ b/userspace/usr/comma/comma.sh @@ -63,7 +63,7 @@ fi # symlink vscode to userdata mkdir -p /data/tmp/vscode-server ln -s /data/tmp/vscode-server ~/.vscode-server - +ln -s /data/tmp/vscode-server ~/.cursor-server while true; do pkill -f "$SETUP" From fd385808dc686b47d4a5edf52aa6a91f2bdc8815 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 28 Jul 2024 13:29:13 -0700 Subject: [PATCH 4/5] cleanup ABL building (#270) --- build_bootloader.sh | 21 --------------------- edk2_tici | 2 +- flash_all.sh | 3 +-- flash_bootloader.sh | 13 ------------- 4 files changed, 2 insertions(+), 37 deletions(-) delete mode 100755 build_bootloader.sh delete mode 100755 flash_bootloader.sh diff --git a/build_bootloader.sh b/build_bootloader.sh deleted file mode 100755 index 542f11f7..00000000 --- a/build_bootloader.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -e - -# Get directories and make sure we're in the correct spot to start the build -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -OUTPUT_DIR=$DIR/output -cd $DIR - -# Clone bootloader if not done already -if [ ! -d edk2_tici ]; then - git submodule init edk2_tici -fi -cd edk2_tici - -# Create output directory -mkdir -p $OUTPUT_DIR - -# Run build -./build.sh - -# Copy output -cp out/* $OUTPUT_DIR/ diff --git a/edk2_tici b/edk2_tici index 67600416..aa3737ce 160000 --- a/edk2_tici +++ b/edk2_tici @@ -1 +1 @@ -Subproject commit 67600416eec240ad754a22707180628b562565d8 +Subproject commit aa3737ce9bb6d215b56ba4238d777042143dbb96 diff --git a/flash_all.sh b/flash_all.sh index cd3e53e9..314ba813 100755 --- a/flash_all.sh +++ b/flash_all.sh @@ -4,11 +4,10 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" cd $DIR -for part in aop xbl xbl_config devcfg; do +for part in aop abl xbl xbl_config devcfg; do tools/edl w ${part}_a $DIR/agnos-firmware/$part.bin tools/edl w ${part}_b $DIR/agnos-firmware/$part.bin done -./flash_bootloader.sh ./flash_kernel.sh ./flash_system.sh diff --git a/flash_bootloader.sh b/flash_bootloader.sh deleted file mode 100755 index 45eb2d2e..00000000 --- a/flash_bootloader.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -e - -GREEN="\033[0;32m" -NO_COLOR='\033[0m' - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR - -echo "Flashing bootloader..." -tools/edl w abl_a output/abl.elf -tools/edl w abl_b output/abl.elf - -echo -e "${GREEN}Flashed abl_a and abl_b!${NO_COLOR}" From c59455d3f843853f0084e9135383e510685c4d25 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 28 Jul 2024 14:34:28 -0700 Subject: [PATCH 5/5] no internal (#271) * no internal * Update README.md * testing * rm private submodules * do this the normal way now * rm agnos-firmware * add size * fix mac * files --- .github/workflows/build.yaml | 7 ---- .gitmodules | 9 ----- README.md | 12 ++++--- TESTING.md | 50 +++++++++++++++++++++++++++ agnos-binaries | 1 - agnos-firmware | 1 - edk2_tici | 1 - firmware.json | 32 +++++++++++++++++ flash_all.sh | 9 ++--- internal/README.md | 51 ---------------------------- internal/build_and_package.sh | 23 ------------- scripts/build_and_package.sh | 14 ++++++++ {internal => scripts}/ota_push.sh | 0 {internal => scripts}/package_ota.py | 32 +++++++++++------ {internal => scripts}/pull_ota.sh | 0 15 files changed, 130 insertions(+), 112 deletions(-) create mode 100644 TESTING.md delete mode 160000 agnos-binaries delete mode 160000 agnos-firmware delete mode 160000 edk2_tici create mode 100644 firmware.json delete mode 100644 internal/README.md delete mode 100755 internal/build_and_package.sh create mode 100755 scripts/build_and_package.sh rename {internal => scripts}/ota_push.sh (100%) rename {internal => scripts}/package_ota.py (80%) rename {internal => scripts}/pull_ota.sh (100%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index edafadd2..49fbdbf9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -43,13 +43,6 @@ jobs: id: kernel-submodule run: echo "ref=$(git ls-tree HEAD | awk '$4 == "agnos-kernel-sdm845"' | awk '{print $3}')" | tee -a $GITHUB_OUTPUT - - name: Checkout agnos-kernel-sdm845 - uses: actions/checkout@v4 - with: - repository: commaai/agnos-kernel-sdm845 - ref: ${{ steps.kernel-submodule.outputs.ref }} - path: agnos-kernel-sdm845 - - name: ccache uses: hendrikmuhs/ccache-action@c92f40bee50034e84c763e33b317c77adaa81c92 with: diff --git a/.gitmodules b/.gitmodules index d91f9a6a..c8520a81 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,3 @@ -[submodule "edk2_tici"] - path = edk2_tici - url = ../../commaai/edk2_tici.git -[submodule "agnos-firmware"] - path = agnos-firmware - url = ../../commaai/agnos-firmware.git [submodule "agnos-kernel-sdm845"] path = agnos-kernel-sdm845 url = ../../commaai/agnos-kernel-sdm845.git -[submodule "agnos-binaries"] - path = agnos-binaries - url = ../../commaai/agnos-binaries.git diff --git a/README.md b/README.md index a1989f10..5bab7416 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,11 @@ This is the builder repo for AGNOS, our Ubuntu based OS. AGNOS runs on the comma three devkit and [comma 3X](https://comma.ai/shop/comma-3x). -NOTE #1: the `edk2_tici` and `agnos-firmware` submodules are internal, private repos. +> [!NOTE] +> Use https://flash.comma.ai if you're looking to reflash your device to a clean factory state. -NOTE #2: Use https://flash.comma.ai if you just want to reflash your device with latest AGNOS version +AGNOS includes all the software to boot and run a comma 3/3X. +This repo builds the two main parts of that (system and kernel) and packages it all up into a "release". ## Development @@ -34,7 +36,7 @@ Flashing to a comma 3/3X: Validating changes: * Running openpilot is a good smoketest for general AGNOS functionality * [CI](https://github.com/commaai/agnos-builder/blob/master/.github/workflows/build.yaml) ensures the kernel and system builds work (and pushes the images for you to download) -* [this](https://github.com/commaai/agnos-builder/blob/master/internal/README.md) is the checklist we go through before shipping new AGNOS releases to openpilot +* [this](https://github.com/commaai/agnos-builder/blob/master/TESTING.md) is the checklist we go through before shipping new AGNOS releases to openpilot ### macOS @@ -74,5 +76,5 @@ Some concrete things on the roadmap: - [ ] update to Ubuntu 24.04 https://github.com/commaai/openpilot/issues/32386 - [ ] mainline Linux kernel https://github.com/commaai/openpilot/issues/32386 - [ ] fully open source - - [ ] anything from `agnos-firmware`: XBL, ABL, etc. - - [ ] open source Weston https://github.com/commaai/agnos-builder/issues/16 + - [ ] anything from [firmware.json](firmware.json): XBL, ABL, etc. + - [ ] Weston https://github.com/commaai/agnos-builder/issues/16 diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 00000000..4fbc83ff --- /dev/null +++ b/TESTING.md @@ -0,0 +1,50 @@ +# testing + +## Release Checklist + +- [ ] [`test_onroad`](https://github.com/commaai/openpilot/blob/master/selfdrive/test/test_onroad.py) passes +- [ ] Wi-Fi: lists networks and connects +- [ ] Modem: connects to cell network +- [ ] Image sizes haven't increased +- [ ] Sounds work +- [ ] `python` is our python, not system version +- [ ] Clean openpilot build: `scons -c && scons -j8` +- [ ] Factory reset + - [ ] from openpilot menu + - [ ] tapping on boot + - [ ] corrupt userdata +- [ ] Color calibration + - [ ] from /persist/comma/ + - [ ] directly from panel over sysfs +- [ ] Clean setup: factory reset -> install openpilot -> openpilot works +- [ ] AGNOS update works on warm boot + - [ ] previous -> new + - [ ] new -> previous + +### ABL + +- [ ] Fastboot USB enumeration +- [ ] Boot time hasn't regressed (3.8s) + +### XBL + +- [ ] Display init works in cold and hot temperatures +- [ ] Boot time hasn't regressed (2.4s) + +### Setup + +#### Networking + +- [ ] Continue button disabled when no connection +- [ ] Forget/connect to wifi + +#### Custom URL +- (a) Not a real URL (e.g. `comma`, `abc123`, `...`) + - [ ] "Ensure the entered URL is valid" + - [ ] Start over + - [ ] Reboot device +- (b) Website but not an installer URL (e.g. `github.com`, `comma.ai`, `installer.comma.ai`) + - [ ] "No custom software found at this URL." +- (c) Valid installer URL (e.g. `openpilot.comma.ai`) + - [ ] Download successful (comma logo or installer appears) + - [ ] `/tmp/installer_url` should contain the installer URL diff --git a/agnos-binaries b/agnos-binaries deleted file mode 160000 index 72a4c036..00000000 --- a/agnos-binaries +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 72a4c0362cc25424d001c05bcab01cca20c231ef diff --git a/agnos-firmware b/agnos-firmware deleted file mode 160000 index 8dbaed43..00000000 --- a/agnos-firmware +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8dbaed43475157aa39736194ecaec62d998ac52c diff --git a/edk2_tici b/edk2_tici deleted file mode 160000 index aa3737ce..00000000 --- a/edk2_tici +++ /dev/null @@ -1 +0,0 @@ -Subproject commit aa3737ce9bb6d215b56ba4238d777042143dbb96 diff --git a/firmware.json b/firmware.json new file mode 100644 index 00000000..8f143eb5 --- /dev/null +++ b/firmware.json @@ -0,0 +1,32 @@ +[ + { + "name": "abl", + "url": "https://commadist.azureedge.net/agnosupdate/abl-eeb89a74c968a5a2ffce96f23158b72e03e2814adf72ef59d1200ba8ea5d2f39.img.xz", + "hash": "eeb89a74c968a5a2ffce96f23158b72e03e2814adf72ef59d1200ba8ea5d2f39", + "size": 274432 + }, + { + "name": "xbl", + "url": "https://commadist.azureedge.net/agnosupdate/xbl-bcef195b00a1ab685da601f4072722569773ab161e91c8753ad99ca4217a28f5.img.xz", + "hash": "bcef195b00a1ab685da601f4072722569773ab161e91c8753ad99ca4217a28f5", + "size": 3282672 + }, + { + "name": "xbl_config", + "url": "https://commadist.azureedge.net/agnosupdate/xbl_config-19791056558c16f8dae787531b5e30b3b3db2ded9d666688df45ce1b91a72bac.img.xz", + "hash": "19791056558c16f8dae787531b5e30b3b3db2ded9d666688df45ce1b91a72bac", + "size": 98124 + }, + { + "name": "devcfg", + "url": "https://commadist.azureedge.net/agnosupdate/devcfg-be44b73dda5be840b09d5347d536459e31098da3fea97596956c0bdad19bdf27.img.xz", + "hash": "be44b73dda5be840b09d5347d536459e31098da3fea97596956c0bdad19bdf27", + "size": 40336 + }, + { + "name": "aop", + "url": "https://commadist.azureedge.net/agnosupdate/aop-5d764611a683d6a738cf06a1dcf8a926d0f47b5117ad40d3054167de6dd8bd0f.img.xz", + "hash": "5d764611a683d6a738cf06a1dcf8a926d0f47b5117ad40d3054167de6dd8bd0f", + "size": 184364 + } +] diff --git a/flash_all.sh b/flash_all.sh index 314ba813..66a39094 100755 --- a/flash_all.sh +++ b/flash_all.sh @@ -4,10 +4,11 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" cd $DIR -for part in aop abl xbl xbl_config devcfg; do - tools/edl w ${part}_a $DIR/agnos-firmware/$part.bin - tools/edl w ${part}_b $DIR/agnos-firmware/$part.bin -done +# TODO: download firmware from firmware.json +#for part in aop abl xbl xbl_config devcfg; do +# tools/edl w ${part}_a $DIR/agnos-firmware/$part.bin +# tools/edl w ${part}_b $DIR/agnos-firmware/$part.bin +#done ./flash_kernel.sh ./flash_system.sh diff --git a/internal/README.md b/internal/README.md deleted file mode 100644 index eece9897..00000000 --- a/internal/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# internal - -## release checklist - -- [ ] `test_onroad` passes -- [ ] wifi -- [ ] modem -- [ ] image size -- [ ] sounds -- [ ] `python` is our python, not system version -- [ ] pyenv shims works -- [ ] clean openpilot build -- [ ] factory reset - - [ ] from openpilot menu - - [ ] tapping on boot - - [ ] corrupt userdata -- [ ] color calibration - - [ ] from /persist/comma/ - - [ ] directly from panel over sysfs -- [ ] clean setup -- [ ] update works on warm boot - - [ ] previous -> new - - [ ] new -> previous - -### ABL - -- [ ] fastboot USB enumeration -- [ ] boot time hasn't regressed (3.8s) - -### XBL - -- [ ] display init works in cold and hot temperatures -- [ ] boot time hasn't regressed (2.4s) - -### setup - -#### networking - -- [ ] continue button disabled when no connection -- [ ] forget/connect to wifi - -#### custom URL -- (a) Not a real URL (e.g. `comma`, `abc123`, `...`) - - [ ] "Ensure the entered URL is valid" - - [ ] Start over - - [ ] Reboot device -- (b) Website but not an installer URL (e.g. `github.com`, `comma.ai`, `installer.comma.ai`) - - [ ] "No custom software found at this URL." -- (c) Valid installer URL (e.g. `openpilot.comma.ai`) - - [ ] Download successful (comma logo or installer appears) - - [ ] `/tmp/installer_url` should contain the installer URL diff --git a/internal/build_and_package.sh b/internal/build_and_package.sh deleted file mode 100755 index 2ec5bf3c..00000000 --- a/internal/build_and_package.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR/.. - -read -p "Is the kernel repo up to date? " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Update it and run again!" - [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 -fi - -read -p "Is the firmware repo up to date? Copied in the new abl if needed? " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Update it and run again!" - [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 -fi - -./build_kernel.sh -./build_system.sh -internal/package_ota.py -internal/ota_push.sh staging diff --git a/scripts/build_and_package.sh b/scripts/build_and_package.sh new file mode 100755 index 00000000..370343af --- /dev/null +++ b/scripts/build_and_package.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd $DIR/.. + +./build_kernel.sh +./build_system.sh +scripts/package_ota.py + +# push to azure on an internal machine +if [ "$USER" == "batman" ]; then + scripts/ota_push.sh staging +fi diff --git a/internal/ota_push.sh b/scripts/ota_push.sh similarity index 100% rename from internal/ota_push.sh rename to scripts/ota_push.sh diff --git a/internal/package_ota.py b/scripts/package_ota.py similarity index 80% rename from internal/package_ota.py rename to scripts/package_ota.py index 0c1bc24c..b0ed8b8e 100755 --- a/internal/package_ota.py +++ b/scripts/package_ota.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json import os +import hashlib import subprocess from copy import deepcopy from pathlib import Path @@ -9,15 +10,16 @@ ROOT = Path(__file__).parent.parent OUTPUT_DIR = ROOT / "output" OTA_OUTPUT_DIR = OUTPUT_DIR / "ota" -FIRMWARE_DIR = ROOT / "agnos-firmware" AGNOS_UPDATE_URL = os.getenv("AGNOS_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate") AGNOS_STAGING_UPDATE_URL = os.getenv("AGNOS_STAGING_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate-staging") - def checksum(fn): - return subprocess.check_output(["sha256sum", fn]).decode().split()[0] - + sha256 = hashlib.sha256() + with open(fn, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + sha256.update(chunk) + return sha256.hexdigest() def compress(fin, fout) -> None: subprocess.check_call(f"xz -T4 -vc {fin} > {fout}", shell=True) @@ -32,7 +34,7 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) if sparse: with NamedTemporaryFile() as tmp_f: print(" converting sparse image to raw") - subprocess.check_call(["simg2img", fn, tmp_f.name]) + subprocess.check_call(["simg2img", fn, tmp_f.name], shell=True) hash_raw = checksum(tmp_f.name) size = Path(tmp_f.name).stat().st_size print(f" {size} bytes, hash {hash} (raw)") @@ -76,11 +78,6 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) files = [ process_file(OUTPUT_DIR / "boot.img", "boot"), - process_file(FIRMWARE_DIR / "abl.bin", "abl"), - process_file(FIRMWARE_DIR / "xbl.bin", "xbl"), - process_file(FIRMWARE_DIR / "xbl_config.bin", "xbl_config"), - process_file(FIRMWARE_DIR / "devcfg.bin", "devcfg"), - process_file(FIRMWARE_DIR / "aop.bin", "aop"), process_file(OUTPUT_DIR / "system.img", "system", sparse=True, full_check=False, alt=OUTPUT_DIR / "system-skip-chunks.img"), ] configs = [ @@ -88,6 +85,21 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) (AGNOS_STAGING_UPDATE_URL, "ota-staging.json"), ] + # pull in firmware not built in this repo + with open(ROOT/"firmware.json") as f: + fws = json.loads(f.read()) + for fw in fws: + files.append({ + "name": fw["name"], + "url": fw["url"], + "hash": fw["hash"], + "hash_raw": fw["hash"], + "size": fw["size"], + "sparse": False, + "full_check": True, + "has_ab": True, + }) + for remote_url, output_fn in configs: processed_files = [] for f in deepcopy(files): diff --git a/internal/pull_ota.sh b/scripts/pull_ota.sh similarity index 100% rename from internal/pull_ota.sh rename to scripts/pull_ota.sh