GGUF converter

gguf-converter/.gitignore

@@ -0,0 +1,2 @@
+params.json
+.venv

gguf-converter/Dockerfile

@@ -0,0 +1,34 @@
+ARG BASE_IMAGE=substratusai/base:latest
+FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 as build
+
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    apt-get update && \
+    apt-get -y --no-install-recommends install \
+      python3 python3-pip python3-venv git build-essential gcc wget && \
+    mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /build
+WORKDIR /build
+RUN git clone https://github.com/ggerganov/llama.cpp.git
+RUN cd llama.cpp  && make quantize
+
+FROM ${BASE_IMAGE}
+
+RUN mkdir -p /content/src /content/data /content/bin
+WORKDIR /content
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+RUN git clone https://github.com/ggerganov/llama.cpp.git
+RUN pip install --no-cache-dir -r llama.cpp/requirements.txt
+COPY --from=build /build/llama.cpp/quantize /content/bin
+
+ENV PATH="$PATH:/content/scripts:/content/bin"
+
+# Copy in build dependencies only since the build will take a while.
+COPY ./scripts/ ./scripts
+COPY ./src/ ./src
+
+ENTRYPOINT ["/tini", "--", "/content/scripts/entrypoint.sh"]
+CMD convert.sh

gguf-converter/README.md

@@ -0,0 +1,24 @@
+# Dataset Loader HTTP
+
+Load existing files into Substratus by downloading using HTTP
+
+## Usage
+
+Build the image locally:
+
+```sh
+docker build -t gguf-converter .
+```
+
+Explore and develop with a Jupyter Lab:
+```sh
+# create test params.json
+cat > params.json <<EOF
+{"name": "test", "download_model_id": "lmsys/vicuna-13b-v1.5"}
+EOF
+# Run a Jupyter Notebook.
+docker run -it -v $(pwd)/src:/content/src \
+  -v $(pwd)/params.json:/content/params.json -p 8888:8888 \
+  --security-opt seccomp=unconfined gguf-converter notebook.sh
+```
+Now open your browser at http://localhost:8888

gguf-converter/requirements.txt

@@ -0,0 +1,5 @@
+jupyterlab
+ipywidgets
+jupytext
+
+huggingface_hub

gguf-converter/scripts/convert.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+set -xe
+
+jupyter nbconvert --debug --to notebook --execute /content/src/convert.ipynb --output /content/logs/convert.ipynb

gguf-converter/scripts/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+export HUGGING_FACE_HUB_TOKEN=$PARAM_HUGGING_FACE_HUB_TOKEN
+
+exec "$@"