diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b983289..d3b484d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,6 +41,19 @@ jobs: LATEST_TAG: ${{ matrix.proc }}-latest steps: + - name: Remove unwanted software + run: | + echo "Available storage before:" + sudo df -h + echo + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + echo "Available storage after:" + sudo df -h + echo + - uses: actions/checkout@v4 - name: Build Image @@ -56,7 +69,9 @@ jobs: - name: Tests packages in container run: | - docker run $LOGIN_SERVER/$IMAGE:$SHA_TAG python -m pytest tests + docker run --network none \ + $LOGIN_SERVER/$IMAGE:$SHA_TAG \ + pixi run -e ${{ matrix.proc }} python -m pytest tests - name: Log into Azure if: ${{ fromJson(env.SHOULD_PUBLISH) }} @@ -78,4 +93,3 @@ jobs: if: ${{ fromJson(env.SHOULD_PUBLISH) }} run: | docker push $LOGIN_SERVER/$IMAGE --all-tags - diff --git a/Makefile b/Makefile index 293f9ff..ab290eb 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ else CPU_OR_GPU ?= gpu endif +BLOCK_INTERNET ?= true + TAG := ${CPU_OR_GPU}-latest LOCAL_TAG := ${CPU_OR_GPU}-local @@ -120,30 +122,31 @@ build: ## Updates runtime environment lockfile using Docker update-lockfile: @echo Generating the lockfile for CPU and GPU within Docker - cd runtime && \ - docker build . \ - --file Dockerfile-lock \ - --build-arg CPU_OR_GPU=${CPU_OR_GPU} \ + docker build runtime \ + --file runtime/Dockerfile-lock \ --tag pixi-lock:local - @echo Copying lockfile to host - docker create --name dummy pixi-lock:local - docker cp dummy:/tmp/pixi.lock runtime/pixi.lock - docker rm -f dummy + @echo Running lock container + docker run \ + --mount type=bind,source="$(shell pwd)"/runtime,target=/tmp \ + --rm \ + pixi-lock:local ## Ensures that your locally built image can import all the Python packages successfully when it runs test-container: _check_image _echo_image _submission_write_perms docker run \ ${GPU_ARGS} \ + ${NETWORK_ARGS} \ ${TTY_ARGS} \ - --mount type=bind,source="$(shell pwd)"/runtime/tests,target=/tests,readonly \ --pid host \ ${SUBMISSION_IMAGE_ID} \ - python -m pytest -v tests + pixi run -e ${CPU_OR_GPU} python -m pytest tests + ## Open an interactive bash shell within the running container (with network access) interact-container: _check_image _echo_image _submission_write_perms docker run \ ${GPU_ARGS} \ + ${NETWORK_ARGS} \ --mount type=bind,source=${shell pwd}/data,target=/code_execution/data,readonly \ --mount type=bind,source="$(shell pwd)/submission",target=/code_execution/submission \ --shm-size 8g \ diff --git a/runtime/Dockerfile b/runtime/Dockerfile index 1dc98af..025ab69 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -10,6 +10,15 @@ ENV DEBIAN_FRONTEND=noninteractive \ PYTHONUNBUFFERED=1 \ SHELL=/bin/bash +# Create user andset permissions +ENV RUNTIME_USER=runtimeuser +ENV RUNTIME_UID=1000 +ENV RUNTIME_GID=1000 + +RUN echo "Creating ${RUNTIME_USER} user..." \ + && groupadd --gid ${RUNTIME_GID} ${RUNTIME_USER} \ + && useradd --create-home --gid ${RUNTIME_GID} --no-log-init --uid ${RUNTIME_UID} ${RUNTIME_USER} + COPY apt.txt apt.txt RUN apt-get update --fix-missing \ && apt-get install -y apt-utils 2> /dev/null \ @@ -17,18 +26,22 @@ RUN apt-get update --fix-missing \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /apt.txt -COPY --chown=$MAMBA_USER:$MAMBA_USER pixi.lock /tmp/pixi.lock -COPY --chown=$MAMBA_USER:$MAMBA_USER pixi.toml /tmp/pixi.toml -RUN pixi install --manifest-path /tmp/pixi.toml -e ${CPU_OR_GPU} && \ - pixi clean --manifest-path /tmp/pixi.toml -e ${CPU_OR_GPU} - +# Set up code execution working directory RUN mkdir /code_execution -RUN chown -R ${MAMBA_USER}:${MAMBA_USER} /code_execution +RUN chown -R ${RUNTIME_USER}:${RUNTIME_USER} /code_execution +WORKDIR /code_execution -COPY tests /code_execution/tests -COPY entrypoint.sh /entrypoint.sh +# Switch to runtime user +USER ${RUNTIME_USER} -WORKDIR /code_execution -USER ${MAMBA_USER} +COPY pixi.lock ./pixi.lock +COPY pixi.toml ./pixi.toml + +RUN pixi install -e ${CPU_OR_GPU} --frozen \ + && pixi clean cache --yes \ + && pixi info + +COPY entrypoint.sh /entrypoint.sh +COPY --chown=${RUNTIME_USER}:${RUNTIME_USER} tests ./tests CMD ["bash", "/entrypoint.sh"] diff --git a/runtime/Dockerfile-lock b/runtime/Dockerfile-lock index d133a82..48e36a7 100644 --- a/runtime/Dockerfile-lock +++ b/runtime/Dockerfile-lock @@ -2,15 +2,7 @@ FROM ghcr.io/prefix-dev/pixi:0.26.1-bookworm-slim USER root -ARG CPU_OR_GPU=gpu - -ENV DEBIAN_FRONTEND=noninteractive \ - LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 \ - PYTHONUNBUFFERED=1 \ - SHELL=/bin/bash - -COPY --chown=$MAMBA_USER:$MAMBA_USER pixi.lock /tmp/pixi.lock -COPY --chown=$MAMBA_USER:$MAMBA_USER pixi.toml /tmp/pixi.toml -RUN pixi tree --manifest-path /tmp/pixi.toml --platform linux-64 +RUN mkdir -p /tmp +WORKDIR /tmp +ENTRYPOINT ["pixi", "tree", "--manifest-path", "pixi.toml", "--platform", "linux-64", "-v"] diff --git a/runtime/entrypoint.sh b/runtime/entrypoint.sh index 538a818..667f2ef 100644 --- a/runtime/entrypoint.sh +++ b/runtime/entrypoint.sh @@ -19,7 +19,7 @@ main () { ls -alh echo "Running submission..." - python main.py + pixi run -e $CPU_OR_GPU python main.py echo "Exporting submission.csv result..."