Use Pixi for dependency management

.github/workflows/build.yml

@@ -41,6 +41,19 @@ jobs:
       LATEST_TAG: ${{ matrix.proc }}-latest
 
     steps:
+      - name: Remove unwanted software
+        run: |
+          echo "Available storage before:"
+          sudo df -h
+          echo
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          echo "Available storage after:"
+          sudo df -h
+          echo
+
       - uses: actions/checkout@v4
 
       - name: Build Image
@@ -56,7 +69,9 @@ jobs:
 
       - name: Tests packages in container
         run: |
-          docker run $LOGIN_SERVER/$IMAGE:$SHA_TAG python -m pytest tests
+          docker run --network none \
+            $LOGIN_SERVER/$IMAGE:$SHA_TAG \
+            /code_execution/.pixi/envs/${{ matrix.proc }}/bin/python -m pytest tests
 
       - name: Log into Azure
         if: ${{ fromJson(env.SHOULD_PUBLISH) }}
@@ -78,4 +93,3 @@ jobs:
         if: ${{ fromJson(env.SHOULD_PUBLISH) }}
         run: |
           docker push $LOGIN_SERVER/$IMAGE --all-tags
-

Makefile

@@ -8,6 +8,8 @@ else
 CPU_OR_GPU ?= gpu
 endif
 
+BLOCK_INTERNET ?= true
+
 TAG := ${CPU_OR_GPU}-latest
 LOCAL_TAG := ${CPU_OR_GPU}-local
 
@@ -109,61 +111,55 @@ endif
 #################################################################################
 # Commands for building the container if you are changing the requirements      #
 #################################################################################
-.PHONY: build clean interact-container pack-example pack-submission pull test-container test-submission update-lockfiles
+.PHONY: build clean interact-container pack-example pack-submission pull test-container test-submission update-lockfile
 
 ## Builds the container locally
 build:
 	docker build runtime \
 		--build-arg CPU_OR_GPU=${CPU_OR_GPU} \
 		--tag ${LOCAL_IMAGE}:${LOCAL_TAG}
 
+runtime/pixi.lock:
+ifeq (,$(wildcard ./runtime/pixi.lock))
+	@echo Generating runtime/pixi.lock from scratch
+	@echo "version: 5" > runtime/pixi.lock
+	@echo "environments:" >> runtime/pixi.lock
+	@echo "packages:" >> runtime/pixi.lock
+else
+	@echo Updating existing runtime/pixi.lock
+endif
+
+## Updates runtime environment lockfile using Docker
+update-lockfile: runtime/pixi.lock
+	@echo Building Docker image to generate lockfile
+	docker build runtime \
+		--file runtime/Dockerfile-lock \
+		--tag pixi-lock:local
+	@echo Running lock container
+	docker run \
+		--mount type=bind,source="$(shell pwd)"/runtime/pixi.toml,target=/tmp/pixi.toml \
+		--mount type=bind,source="$(shell pwd)"/runtime/pixi.lock,target=/tmp/pixi.lock \
+		--rm \
+		pixi-lock:local
 
-runtime/conda-lock-cpu.yml: runtime/environment-cpu.yml
-	@echo Locking the CPU environment
-	conda-lock \
-		--mamba \
-		-p linux-64 \
-		--without-cuda \
-		-f runtime/environment-cpu.yml \
-		--lockfile runtime/conda-lock-cpu.yml
-	conda-lock render \
-		-p linux-64 \
-		--filename-template runtime/environment-cpu.lock \
-		runtime/conda-lock-cpu.yml
-
-runtime/conda-lock-gpu.yml: runtime/environment-gpu.yml
-	@echo Locking the GPU environment
-	conda-lock \
-		--mamba \
-		-p linux-64 \
-		--with-cuda 11.8 \
-		-f runtime/environment-gpu.yml \
-		--lockfile runtime/conda-lock-gpu.yml
-	conda-lock render \
-		-p linux-64 \
-		--filename-template runtime/environment-gpu.lock \
-		runtime/conda-lock-gpu.yml
-
-## Updates runtime environment lockfiles
-update-lockfiles: runtime/conda-lock-cpu.yml runtime/conda-lock-gpu.yml
-	@python runtime/tests/test_lockfile.py runtime/conda-lock-cpu.yml
-	@python runtime/tests/test_lockfile.py runtime/conda-lock-gpu.yml
 
 
 ## Ensures that your locally built image can import all the Python packages successfully when it runs
 test-container: _check_image _echo_image _submission_write_perms
 	docker run \
 		${GPU_ARGS} \
+		${NETWORK_ARGS} \
 		${TTY_ARGS} \
-		--mount type=bind,source="$(shell pwd)"/runtime/tests,target=/tests,readonly \
 		--pid host \
 		${SUBMISSION_IMAGE_ID} \
-		python -m pytest -v tests
+		pixi run -e ${CPU_OR_GPU} python -m pytest tests
+
 
 ## Open an interactive bash shell within the running container (with network access)
 interact-container: _check_image _echo_image _submission_write_perms
 	docker run \
 		${GPU_ARGS} \
+		${NETWORK_ARGS} \
 		--mount type=bind,source=${shell pwd}/data,target=/code_execution/data,readonly \
 		--mount type=bind,source="$(shell pwd)/submission",target=/code_execution/submission \
 		--shm-size 8g \

runtime/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 mambaorg/micromamba:1.5.3-bookworm-slim
+FROM ghcr.io/prefix-dev/pixi:0.26.1-jammy-cuda-11.8.0
 
 USER root
 
@@ -10,24 +10,39 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PYTHONUNBUFFERED=1 \
     SHELL=/bin/bash
 
+# Create user andset permissions
+ENV RUNTIME_USER=runtimeuser
+ENV RUNTIME_UID=1000
+ENV RUNTIME_GID=1000
+ENV CPU_OR_GPU=$CPU_OR_GPU
+
+RUN echo "Creating ${RUNTIME_USER} user..." \
+    && groupadd --gid ${RUNTIME_GID} ${RUNTIME_USER}  \
+    && useradd --create-home --gid ${RUNTIME_GID} --no-log-init --uid ${RUNTIME_UID} ${RUNTIME_USER}
+
 COPY apt.txt apt.txt
 RUN apt-get update --fix-missing \
     && apt-get install -y apt-utils 2> /dev/null \
     && xargs -a apt.txt apt-get install -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /apt.txt
 
-COPY --chown=$MAMBA_USER:$MAMBA_USER conda-lock-${CPU_OR_GPU}.yml /tmp/conda-lock.yml
-RUN micromamba install --name base --yes --file /tmp/conda-lock.yml && \
-    micromamba clean --all --force-pkgs-dirs --yes
-
+# Set up code execution working directory
 RUN mkdir /code_execution
-RUN chown -R ${MAMBA_USER}:${MAMBA_USER} /code_execution
+RUN chown -R ${RUNTIME_USER}:${RUNTIME_USER} /code_execution
+WORKDIR /code_execution
 
-COPY tests /code_execution/tests
-COPY entrypoint.sh /entrypoint.sh
+# Switch to runtime user
+USER ${RUNTIME_USER}
 
-WORKDIR /code_execution
-USER ${MAMBA_USER}
+COPY pixi.lock ./pixi.lock
+COPY pixi.toml ./pixi.toml
+
+RUN pixi install -e ${CPU_OR_GPU} --frozen \
+    && pixi clean cache --yes \
+    && pixi info
+
+COPY entrypoint.sh /entrypoint.sh
+COPY --chown=${RUNTIME_USER}:${RUNTIME_USER} tests ./tests
 
 CMD ["bash", "/entrypoint.sh"]

runtime/Dockerfile-lock

@@ -0,0 +1,8 @@
+FROM ghcr.io/prefix-dev/pixi:0.26.1-jammy
+
+USER root
+
+RUN mkdir -p /tmp
+WORKDIR /tmp
+
+ENTRYPOINT ["pixi", "tree", "--manifest-path", "pixi.toml", "--platform", "linux-64", "-v"]