diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..636c464 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +# Docs: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + reviewers: + - "gardenlinux/garden-linux-maintainers" \ No newline at end of file diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..40e4358 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,21 @@ +# Garden Linux Builder CI Workflows + +## `build.yml` + +Build container images on all branches. + +For pushes on the `main` branch, tags based on the git sha are created and pushed to the container registry and a pseudo-release called `latest` is updated on GitHub. +This allows users to follow a rolling-release approach if they desire. + +## `release.yml` + +Tag container images and create GitHub Releases. +This workflow only runs on demand (workflow dispatch). +It should be run if a new release is desired. +The workflow dispatch needs a parameter `component` which specifies which version component should be increased. +This is either `minor` (the default) or `major`. +`major` should be picked in cases where the new version has breaking changes (for example between the `build` script and the container image). + +## `differential-shellcheck.yml` + +Finds new warnings using [shellcheck](https://www.shellcheck.net) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 156a742..7bc5df7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,13 +1,16 @@ -on: push +name: Build +on: + push: + workflow_dispatch: jobs: build: runs-on: ubuntu-latest steps: - - uses: nkraetzschmar/workflow-telemetry-action@v1 + - uses: gardenlinux/workflow-telemetry-action@v1 with: metric_frequency: 1 comment_on_pr: false - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: build run: | sudo apt-get update @@ -26,12 +29,14 @@ jobs: with: name: build path: build - release: + + # Run for new commits on the main branch + release-latest: runs-on: ubuntu-latest needs: build if: github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: tag latest run: | git tag --force latest @@ -42,5 +47,5 @@ jobs: path: download - name: create release run: | - release="$(.github/workflows/release.sh ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} create latest Builder)" + release="$(.github/workflows/release.sh ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} create latest "Builder (latest)")" .github/workflows/release.sh ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} upload "$release" download/build diff --git a/.github/workflows/bump.py b/.github/workflows/bump.py new file mode 100755 index 0000000..cc47d07 --- /dev/null +++ b/.github/workflows/bump.py @@ -0,0 +1,96 @@ +#!/usr/bin/python + +""" +Determine next version number for versions of a schema like v1.0 +based on existing git tags and which component to bump (minor/major). +""" + +import subprocess +import re +import sys +import os + + +def convert_version_to_sortable_int(major, minor): + return major * 1000 + minor + + +def determine_most_recent_existing_version(): + tags = subprocess.run(["git", "tag"], capture_output=True).stdout.splitlines() + + versions = [] + + for t in tags: + tag = t.decode() + if re.match(r"v[0-9]+.[0-9]+", tag): + tag_without_prefix = tag[1:] + components = tag_without_prefix.split(".") + assert len(components) == 2 + major_int = int(components[0]) + minor_int = int(components[1]) + versions.append( + { + "tag": tag, + "sortNumber": convert_version_to_sortable_int(major_int, minor_int), + "major": major_int, + "minor": minor_int, + } + ) + + if len(versions) == 0: + print("No existing versions found") + return { + "tag": "v0.0", + "sortNumber": convert_version_to_sortable_int(0, 0), + "major": 0, + "minor": 0, + } + + def keyToSortVersions(v): + return v["sortNumber"] + + versions.sort(key=keyToSortVersions, reverse=True) + print(f"Sorted list of versions: {versions}") + highest_existing_version_number = versions[0] + + return highest_existing_version_number + + +def bump(most_recent_version, component_to_bump): + new_version = "" + + if component_to_bump == "major": + new_major = most_recent_version["major"] + 1 + new_version = f"v{new_major}.0" + elif component_to_bump == "minor": + new_minor = most_recent_version["minor"] + 1 + major = most_recent_version["major"] + new_version = f"v{major}.{new_minor}" + else: + raise ( + f"Invalid component provided: {component_to_bump}, only major or minor are supported." + ) + + return new_version + + +def determine_component_to_bump(): + if sys.argv[1] not in ["major", "minor"]: + raise ("Usage: bump.py (major|minor)") + return sys.argv[1] + + +def main(): + component_to_bump = determine_component_to_bump() + most_recent_version = determine_most_recent_existing_version() + new_version = bump(most_recent_version, component_to_bump) + + if os.getenv("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as file_handle: + print(f"newVersion={new_version}", file=file_handle) + else: + print(f"No GitHub env found. New version is {new_version}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/differential-shellcheck.yml b/.github/workflows/differential-shellcheck.yml new file mode 100644 index 0000000..cfe811b --- /dev/null +++ b/.github/workflows/differential-shellcheck.yml @@ -0,0 +1,37 @@ +name: Differential ShellCheck +on: + push: + branches: + - main + - rel-* + pull_request: + branches: + - main + - rel-* + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read + + steps: + - name: Repository checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # pin@v4.1.1 + with: + fetch-depth: 0 + + - id: ShellCheck + name: Differential ShellCheck + uses: redhat-plumbers-in-action/differential-shellcheck@91e2582e40236f831458392d905578d680baa138 # pin@aa647ec4466543e8555c2c3b648124a9813cee44 + with: + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..76605d6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Release +on: + workflow_dispatch: + inputs: + component: + description: 'Version component to increment (Use *minor* unless we have breaking changes)' + required: true + type: choice + options: + - minor + - major +jobs: + release-new-version: + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' && github.event.inputs.component != '' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: echo Version Component to Increase is ${{ github.event.inputs.component }} + - name: get next version number + run: .github/workflows/bump.py ${{ github.event.inputs.component }} + id: bump + - run: echo New version number ${{ steps.bump.outputs.newVersion }} + - name: tag container image + run: | + SHA=$(git rev-parse HEAD) + podman login -u token -p ${{ github.token }} ghcr.io + podman pull ghcr.io/${{ github.repository }}:amd64-"$SHA" + podman pull ghcr.io/${{ github.repository }}:arm64-"$SHA" + podman manifest create ghcr.io/${{ github.repository }}:${{ steps.bump.outputs.newVersion }} + podman manifest add ghcr.io/${{ github.repository }}:${{ steps.bump.outputs.newVersion }} ghcr.io/${{ github.repository }}:amd64-"$SHA" + podman manifest add ghcr.io/${{ github.repository }}:${{ steps.bump.outputs.newVersion }} ghcr.io/${{ github.repository }}:arm64-"$SHA" + podman manifest push ghcr.io/${{ github.repository }}:${{ steps.bump.outputs.newVersion }} + sed -i 's|container_image=localhost/builder|container_image=ghcr.io/${{ github.repository }}:${{ steps.bump.outputs.newVersion }}|' build + - name: git tag + run: | + git tag ${{ steps.bump.outputs.newVersion }} + git push origin ${{ steps.bump.outputs.newVersion }} + - name: create release (new version) + run: | + release="$(.github/workflows/release.sh ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} create ${{ steps.bump.outputs.newVersion }} "Builder (${{ steps.bump.outputs.newVersion }})")" + .github/workflows/release.sh ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} upload "$release" download/build diff --git a/Dockerfile b/Dockerfile index ba3c4cc..60c19cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,28 @@ -FROM debian:trixie AS mv_data +FROM debian:testing AS mv_data RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends build-essential ca-certificates git -RUN git clone --depth=1 https://github.com/nkraetzschmar/mv_data +RUN git clone --depth=1 https://github.com/gardenlinux/mv_data RUN make -C mv_data install -FROM debian:trixie AS aws-kms-pkcs11 +FROM debian:testing AS aws-kms-pkcs11 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends build-essential awscli ca-certificates cmake git libcurl4-openssl-dev libengine-pkcs11-openssl libjson-c-dev libssl-dev libp11-kit-dev libp11-dev zlib1g-dev RUN git clone --depth=1 --recurse-submodules -b 1.11.25 https://github.com/aws/aws-sdk-cpp RUN mkdir aws-sdk-cpp/.build && cd aws-sdk-cpp/.build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="kms;acm-pca" .. && make -j "$(nproc)" install -RUN git clone --depth=1 -b v0.0.10 https://github.com/JackOfMostTrades/aws-kms-pkcs11 +RUN git clone --depth=1 -b v0.0.10 https://github.com/gardenlinux/aws-kms-pkcs11 RUN cd aws-kms-pkcs11 && make -j "$(nproc)" AWS_SDK_STATIC=y install RUN cp "/usr/lib/$(uname -m)-linux-gnu/pkcs11/aws_kms_pkcs11.so" /aws_kms_pkcs11.so -FROM debian:trixie +FROM debian:testing + +LABEL org.opencontainers.image.source="https://github.com/gardenlinux/builder" +LABEL org.opencontainers.image.description="Builder for Garden Linux" + COPY pkg.list /pkg.list -RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends $(cat /pkg.list) && rm /pkg.list +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $(cat /pkg.list) && rm /pkg.list COPY --from=mv_data /usr/bin/mv_data /usr/bin/mv_data COPY --from=aws-kms-pkcs11 /aws_kms_pkcs11.so /aws_kms_pkcs11.so RUN mv /aws_kms_pkcs11.so "/usr/lib/$(uname -m)-linux-gnu/pkcs11/aws_kms_pkcs11.so" COPY builder /builder RUN mkdir /builder/cert COPY setup_namespace /usr/sbin/setup_namespace -RUN echo 'root:0:65536' | tee /etc/subuid /etc/subgid > /dev/null +RUN echo 'root:1:65535' | tee /etc/subuid /etc/subgid > /dev/null ENTRYPOINT [ "/usr/sbin/setup_namespace" ] diff --git a/build b/build index d1abd1e..309cb4f 100755 --- a/build +++ b/build @@ -75,7 +75,14 @@ done if [ "$container_image" = localhost/builder ]; then dir="$(dirname -- "$(realpath -- "${BASH_SOURCE[0]}")")" - "$container_engine" build -t "$container_image" "$dir" + # Build from 'builder.dockerfile' if that exists, otherwise the default file name will be 'Dockerfile' or 'Containerfile'. + # It is recommended to call the file 'builder.dockerfile' to make it's intention clear. + # That file might only contain a single line 'FROM ghcr.io/gardenlinux/builder:...' which can be updated via dependabot. + if [[ -f builder.dockerfile ]]; then + "$container_engine" build -t "$container_image" -f builder.dockerfile "$dir" + else + "$container_engine" build -t "$container_image" "$dir" + fi fi repo="$(./get_repo)" diff --git a/builder/Makefile b/builder/Makefile index 6cb75e8..5adf649 100644 --- a/builder/Makefile +++ b/builder/Makefile @@ -95,5 +95,5 @@ $(foreach artifact_rule,$(shell ./make_get_artifact_rules),$(eval $(call artifac ln -f -s -r '$<' '.build/$*' # prevents match anything rule from applying to Makefile and image/convert scripts -Makefile image image.release image.manifest $(shell find features -name 'convert.*' -o -name 'image.*'): +Makefile image image.release image.manifest $(shell find features -name 'convert.*' -o -name image -o -name 'image.*'): true diff --git a/builder/image.d/makepart b/builder/image.d/makepart index bb18376..6386af5 100755 --- a/builder/image.d/makepart +++ b/builder/image.d/makepart @@ -50,7 +50,7 @@ sed 's/#.*//;/^[[:space:]]*$/d' \ resize=1 verity=0 secureboot=0 - syslinux=$([[ "$(cut -c -5 <<< "$target")" = "/boot" ]] && [[ -f "$rootfs/usr/bin/syslinux" ]] && echo 1 || echo 0) + syslinux=$([[ "$(cut -c -5 <<< "$target")" = "/boot" ]] || [[ "$(tr -d '[:blank:]' <<< "$target")" = "/efi" ]] && [[ -f "$rootfs/usr/bin/syslinux" ]] && echo 1 || echo 0) ephemeral=0 ephemeral_cryptsetup=0 weight=1 diff --git a/builder/image.d/makesecureboot b/builder/image.d/makesecureboot index a616a2f..b362849 100755 --- a/builder/image.d/makesecureboot +++ b/builder/image.d/makesecureboot @@ -93,24 +93,20 @@ case "$BUILDER_ARCH" in amd64) uefi_arch=X64 gnu_arch=x86_64 - initrd_vma=0x3000000 ;; arm64) uefi_arch=AA64 gnu_arch=aarch64 - initrd_vma=0x4000000 ;; esac # create unified image -cmdline_file=$(mktemp) -echo "$cmdline" > "$cmdline_file" -"${gnu_arch}-linux-gnu-objcopy" \ - --add-section .cmdline="$cmdline_file" --change-section-vma .cmdline=0x1000000 \ - --add-section .linux="$kernel_file" --change-section-vma .linux=0x2000000 \ - --add-section .initrd="$initrd" --change-section-vma .initrd="$initrd_vma" \ - "$rootfs/usr/lib/systemd/boot/efi/linux$(tr '[:upper:]' '[:lower:]' <<< "$uefi_arch").efi.stub" "$unified_image" -rm "$cmdline_file" +/usr/lib/systemd/ukify build \ + --stub "$rootfs/usr/lib/systemd/boot/efi/linux$(tr '[:upper:]' '[:lower:]' <<< "$uefi_arch").efi.stub" \ + --linux "$kernel_file" \ + --initrd "$initrd" \ + --cmdline "$cmdline" \ + --output "$unified_image" efi_dir="$(mktemp -d)" mkdir -p "$efi_dir/EFI/BOOT/" diff --git a/builder/parse_features b/builder/parse_features index e293d12..861ac3f 100755 --- a/builder/parse_features +++ b/builder/parse_features @@ -23,6 +23,7 @@ def main(): args_type_allowed = [ "cname", + "cname_base", "features", "platforms", "flags", @@ -35,6 +36,10 @@ def main(): args = parser.parse_args() assert bool(args.features) ^ bool(args.cname), "please provide either `--features` or `--cname` argument" + + arch = None + version = None + if args.cname: search = re.search("^([a-z][a-zA-Z0-9_-]*?)(-(amd64|arm64)(-([a-z0-9.]+))?)?$", args.cname) assert search, f"not a valid cname {args.cname}" @@ -48,6 +53,7 @@ def main(): if args.arch: arch = args.arch + if args.version: version = args.version @@ -90,7 +96,9 @@ def main(): cname_base = get_cname_base(sorted_minimal_features) cname = f"{cname_base}-{arch}-{version}" - if args.type == "cname": + if args.type == "cname_base": + print(cname_base) + elif args.type == "cname": print(cname) elif args.type == "features": print(",".join(features)) @@ -160,6 +168,7 @@ def read_feature_files(feature_dir): if attr not in [ "include", "exclude" ]: continue for ref in node_features[attr]: + assert os.path.isfile(f"{feature_dir}/{ref}/info.yaml"), f"feature {node} references feature {ref}, but {feature_dir}/{ref}/info.yaml does not exist" feature_graph.add_edge(node, ref, attr=attr) assert networkx.is_directed_acyclic_graph(feature_graph) return feature_graph diff --git a/docs/getting_started.md b/docs/getting_started.md index 87b2337..f49a189 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -2,11 +2,17 @@ This tutorial will walk you through the process of creating a custom Linux image using the Builder tool. We will start with the Builder example repository and build a feature to add an `nginx` HTTP server to our image. -Let's begin by cloning the Builder example repository: +Let's begin by creating a new GitHub repository based on the Builder example repository using this link: + +https://github.com/new?template_name=builder_example&template_owner=gardenlinux + +This repo has a GitHub Actions workflow enabled, so it will already start building the image on GitHub's hosted runners. + +To customize the image, clone the repo locally: ```shell -git clone https://github.com/gardenlinux/builder_example -cd builder_example +git clone https://github.com/your_username/my_linux_image +cd my_linux_image ``` To ensure that your local Podman installation is working correctly, you can test it by running the following command: @@ -91,4 +97,27 @@ qemu-system-x86_64 -m 2048 -nodefaults -display none -serial mon:stdio -drive if If everything worked as intended, you should see the system boot up. Once the system is booted, opening http://localhost:8080 in a browser should display the "Hello World!" message. -Congratulations! You have successfully created your first feature for the Builder. :tada: +To also build the new image on GitHub Actions, we'll need to modify the `.github/workflows/build.yml` file. + +Let's change the *build* step to include the `nginx` feature we just created, and let's upload our built image to GitHub's artifact storage: + +```diff +diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml +index 181a646..9e4261e 100644 +--- a/.github/workflows/build.yml ++++ b/.github/workflows/build.yml +@@ -13,4 +13,8 @@ jobs: + steps: + - uses: actions/checkout@v3 + - name: Build the image +- run: ./build base ++ run: ./build base-nginx ++ - uses: actions/upload-artifact@v3 ++ with: ++ name: my-linux-image ++ path: .build/ +``` + +Now commit and push your changes and GitHub will build the image for you. + +Congratulations! You have successfully created your first feature for the Builder and setup a CI Pipeline to build the image. :tada: diff --git a/pkg.list b/pkg.list index 05e6d28..191aeaa 100644 --- a/pkg.list +++ b/pkg.list @@ -1,3 +1,4 @@ +binutils binutils-aarch64-linux-gnu binutils-x86-64-linux-gnu bsdextrautils @@ -7,6 +8,7 @@ cryptsetup curl datefudge debootstrap +dosfstools e2fsprogs fdisk gnupg2 @@ -15,13 +17,17 @@ libengine-pkcs11-openssl libjson-c5 make mtools +ostree +ostree-boot python3 python3-mako python3-networkx +python3-pefile python3-yaml qemu-utils sbsigntool squashfs-tools systemd uidmap +xorriso xz-utils