-
Notifications
You must be signed in to change notification settings - Fork 44
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Split caikit-tgis into separate containers (#107)
* add caikit-only dockerfile - create caikit image using UBI as base - use poetry to lock deps instead of pipenv * docs: update caikit-tgis example * remove smoketest * cleanup kserve docs * update image names/tags
- Loading branch information
Showing
14 changed files
with
3,379 additions
and
2,480 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
.env | ||
.venv | ||
env/ | ||
venv/ | ||
ENV/ | ||
env.bak/ | ||
venv.bak/ | ||
|
||
.mypy_cache/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
poetry.lock -diff |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,31 +1,40 @@ | ||
FROM quay.io/opendatahub/text-generation-inference:stable-bafd218 | ||
FROM registry.access.redhat.com/ubi8/ubi-minimal:latest as poetry-builder | ||
|
||
USER root | ||
RUN microdnf -y update && \ | ||
microdnf -y install \ | ||
shadow-utils python39-pip python39-wheel && \ | ||
pip3 install --no-cache-dir --upgrade pip wheel && \ | ||
microdnf clean all | ||
|
||
# Add grpc-ecosystem health probe | ||
ARG GRPC_HEALTH_PROBE_VERSION=v0.4.20 | ||
ENV POETRY_VIRTUALENVS_IN_PROJECT=1 | ||
|
||
WORKDIR /tmp/poetry | ||
COPY pyproject.toml . | ||
COPY poetry.lock . | ||
RUN pip3 install poetry && poetry install | ||
|
||
|
||
FROM registry.access.redhat.com/ubi8/ubi-minimal:latest as deploy | ||
RUN microdnf -y update && \ | ||
microdnf -y install \ | ||
shadow-utils python39 && \ | ||
microdnf clean all | ||
|
||
WORKDIR /caikit | ||
COPY caikit /caikit | ||
|
||
RUN yum -y update && yum -y install git git-lfs && yum clean all && \ | ||
git lfs install && \ | ||
pip install 'micropipenv[toml]' && \ | ||
micropipenv install && \ | ||
rm -rf ~/.cache && \ | ||
mkdir -p /opt/models && \ | ||
adduser -g 0 -u 1001 caikit --home-dir /caikit && \ | ||
chown -R 1001:0 /caikit /opt/models && \ | ||
chmod -R g=u /caikit /opt/models | ||
COPY --from=poetry-builder /tmp/poetry/.venv /caikit/ | ||
|
||
ENV VIRTUAL_ENV=/caikit | ||
ENV PATH="$VIRTUAL_ENV/bin:$PATH" | ||
|
||
# This is for the use-cases without kserve | ||
RUN curl -Lo /usr/local/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ | ||
chmod +x /usr/local/bin/grpc_health_probe | ||
RUN groupadd --system caikit --gid 1001 && \ | ||
adduser --system --uid 1001 --gid 0 --groups caikit \ | ||
--create-home --home-dir /caikit --shell /sbin/nologin \ | ||
--comment "Caikit User" caikit | ||
|
||
USER 1001 | ||
USER caikit | ||
|
||
ENV TRANSFORMERS_CACHE="/tmp/transformers_cache" \ | ||
RUNTIME_LIBRARY='caikit_nlp' \ | ||
RUNTIME_LOCAL_MODELS_DIR='/opt/models' | ||
ENV CONFIG_FILES=/caikit/config/caikit.yml | ||
VOLUME ["/caikit/config/"] | ||
|
||
CMD [ "./start-serving.sh" ] | ||
CMD ["python", "-m", "caikit.runtime.grpc_server"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,39 +1,26 @@ | ||
TGIS_IMAGE=quay.io/opendatahub/text-generation-inference:stable-bafd218 | ||
CAIKIT_IMAGE=quay.io/opendatahub/caikit-tgis-serving | ||
|
||
.PHONY: default | ||
.PHONY: default refresh-poetry-lock-files | ||
|
||
default: | ||
podman build -t caikit-tgis-serving:$$(git rev-parse --short HEAD) . | ||
podman build \ | ||
-t $(CAIKIT_IMAGE):dev \ | ||
-t $(CAIKIT_IMAGE):$$(git rev-parse --short HEAD) \ | ||
. | ||
|
||
|
||
.PHONY: refresh-piplock-files | ||
|
||
refresh-piplock-files: | ||
podman run --user root -it \ | ||
--rm -v $$(pwd)/caikit:/app/caikit:z \ | ||
$(TGIS_IMAGE) \ | ||
refresh-poetry-lock-files: default | ||
podman run --user root -it --rm \ | ||
--volume $$(pwd):/app:z \ | ||
--workdir /app \ | ||
$(CAIKIT_IMAGE):dev \ | ||
/bin/bash -c " \ | ||
cd caikit && \ | ||
yum -y install git && pip install pipenv && \ | ||
pipenv lock --pre \ | ||
pip install poetry && \ | ||
poetry update \ | ||
" | ||
|
||
|
||
.PHONY: docker-test | ||
|
||
docker-test: default | ||
podman run -it --rm \ | ||
--name caikit-tgis-serving-test-$$(git rev-parse --short HEAD) \ | ||
--volume $$(pwd)/test:/tmp/test:z --volume $$(pwd)/utils:/tmp/utils:z \ | ||
caikit-tgis-serving:$$(git rev-parse --short HEAD) \ | ||
/tmp/test/smoke-test.sh | ||
|
||
|
||
.PHONY: shell | ||
|
||
shell: default | ||
podman run -it --rm \ | ||
--name caikit-tgis-serving-test-$$(git rev-parse --short HEAD) \ | ||
--volume $$(pwd)/test:/tmp/test:z --volume $$(pwd)/utils:/tmp/utils:z \ | ||
caikit-tgis-serving:$$(git rev-parse --short HEAD) \ | ||
$(CAIKIT_IMAGE):dev \ | ||
/bin/bash |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,39 +1,38 @@ | ||
# Bootstrap process (optional) | ||
|
||
Caikit-TGIS image always requires a Caikit-formatted model. Below are the instructions from converting a model to a Caikit-formatted one. | ||
Caikit requires equires a Caikit-formatted model. Below are the instructions from converting a model to a Caikit-formatted one. | ||
|
||
1. Clone the model repository (or have the model folder in a directory). In the below example, Bloom-560m model repo is cloned. | ||
~~~ | ||
|
||
```bash | ||
yum -y install git git-lfs | ||
git lfs install | ||
git clone https://huggingface.co/bigscience/bloom-560m | ||
~~~ | ||
``` | ||
|
||
2. Clone the caikit-nlp repo: | ||
~~~ | ||
git clone https://github.com/caikit/caikit-nlp.git | ||
~~~ | ||
2. Create a virtual environment with Python 3.9 and install `caikit-nlp` | ||
|
||
3. Create a virtual environment with Python 3.9 and install the caikit-nlp | ||
~~~ | ||
python3 -m virtualenv -p python3.9 venv | ||
```bash | ||
python3 -m venv -p python3.9 venv | ||
source venv/bin/activate | ||
python3.9 -m pip install ./caikit-nlp | ||
~~~ | ||
python3.9 -m pip install git+https://github.com/caikit/caikit-nlp.git | ||
``` | ||
|
||
3. (Optional) Clone the `caikit-tgis-serving` repo, if not already available. | ||
|
||
4. (Optional) Clone the caikit-tgis-serving repo, if not already available. | ||
~~~ | ||
```bash | ||
git clone https://github.com/opendatahub-io/caikit-tgis-serving.git | ||
~~~ | ||
``` | ||
|
||
4. Invoke the conversion script located in /utils of caikit-tgis-serving repo. | ||
|
||
```bash | ||
caikit-tgis-serving/utils/convert.py --model-path ./bloom-560m/ --model-save-path ./bloom-560m-caikit | ||
``` | ||
|
||
5. Invoke the conversion script located in /utils of caikit-tgis-serving repo. | ||
~~~ | ||
cp caikit-tgis-serving/utils/convert.py . | ||
./convert.py --model-path ./bloom-560m/ --model-save-path ./bloom-560m-caikit | ||
~~~ | ||
5. Move the model folder (ie. `/bloom-560m-caikit`) into desired storage (ie. S3, MinIO, PVC or other) | ||
6. Do **not** include the model folder name/directory directly in `InferenceService`, but rather point to the directory where the model folder is located. Let's say the `bloom-560m-caikit` directory is located at: `example-models/llm/models/bloom-560m-caikit/`, then `storageUri` value in the InferenceService CR should look like: | ||
|
||
6. Move the model folder (ie. `/bloom-560m-caikit`) into desired storage (ie. S3, MinIO, PVC or other) | ||
7. Do *not* include the model folder name/directory directly in InferenceService, but rather point to the directory where the model folder is located. Let's say the `bloom-560m-caikit` directory is located at: `example-models/llm/models/bloom-560m-caikit/`, then `storageUri` value in the InferenceService CR should look like: | ||
~~~ | ||
```bash | ||
storageUri: s3://example-models/llm/models | ||
~~~ | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,26 +1,73 @@ | ||
--- | ||
apiVersion: v1 | ||
kind: ConfigMap | ||
metadata: | ||
name: caikit-tgis-config | ||
data: | ||
caikit.yml: | | ||
runtime: | ||
library: caikit_nlp | ||
local_models_dir: /mnt/models/ | ||
lazy_load_local_models: true | ||
model_management: | ||
finders: | ||
default: | ||
type: MULTI | ||
config: | ||
finder_priority: | ||
- tgis-auto | ||
tgis-auto: | ||
type: TGIS-AUTO | ||
config: | ||
test_connection: true | ||
initializers: | ||
default: | ||
type: LOCAL | ||
config: | ||
backend_priority: | ||
- type: TGIS | ||
config: | ||
connection: | ||
hostname: localhost:8033 | ||
--- | ||
apiVersion: serving.kserve.io/v1alpha1 | ||
kind: ServingRuntime | ||
metadata: | ||
name: caikit-runtime | ||
spec: | ||
containers: | ||
- env: | ||
- name: RUNTIME_LOCAL_MODELS_DIR | ||
value: /mnt/models | ||
# TODO: This will eventually point to the official image | ||
image: quay.io/opendatahub/caikit-tgis-serving@sha256:20cbc7b312e0dc63903d4a2c7d70b741135ffea31113af63dd8691da4baae0f9 | ||
name: kserve-container | ||
ports: | ||
# Note, KServe only allows a single port, this is the gRPC port. Subject to change in the future | ||
- containerPort: 8085 | ||
name: h2c | ||
protocol: TCP | ||
resources: | ||
requests: | ||
cpu: 8 | ||
memory: 16Gi | ||
multiModel: false | ||
supportedModelFormats: | ||
# Note: this currently *only* supports caikit format models | ||
- autoSelect: true | ||
name: caikit | ||
# Note: this currently *only* supports caikit format models | ||
- autoSelect: true | ||
name: caikit | ||
containers: | ||
- name: kserve-container | ||
image: quay.io/opendatahub/text-generation-inference:stable | ||
command: ["text-generation-launcher"] | ||
args: ["--model-name=/mnt/models/artifacts/"] | ||
env: | ||
- name: TRANSFORMERS_CACHE | ||
value: /tmp/transformers_cache | ||
# resources: # configure as required | ||
# requests: | ||
# cpu: 8 | ||
# memory: 16Gi | ||
- name: transformer-container | ||
image: quay.io/opendatahub/caikit-tgis-serving:stable | ||
volumeMounts: | ||
- name: config-volume | ||
mountPath: /caikit/config/ | ||
readOnly: true | ||
ports: | ||
- containerPort: 8085 | ||
name: h2c | ||
protocol: TCP | ||
# resources: # configure as required | ||
# requests: | ||
# cpu: 8 | ||
# memory: 16Gi | ||
volumes: | ||
- name: config-volume | ||
configMap: | ||
name: caikit-tgis-config |
Oops, something went wrong.