Split caikit-tgis into separate containers

.dockerignore

@@ -0,0 +1,9 @@
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+.mypy_cache/

.gitattributes

@@ -0,0 +1 @@
+poetry.lock -diff

Dockerfile

@@ -1,31 +1,40 @@
-FROM quay.io/opendatahub/text-generation-inference:stable-bafd218
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest as poetry-builder
 
-USER root
+RUN microdnf -y update && \
+    microdnf -y install \
+        shadow-utils python39-pip python39-wheel && \
+    pip3 install --no-cache-dir --upgrade pip wheel && \
+    microdnf clean all
 
-# Add grpc-ecosystem health probe
-ARG GRPC_HEALTH_PROBE_VERSION=v0.4.20
+ENV POETRY_VIRTUALENVS_IN_PROJECT=1
+
+WORKDIR /tmp/poetry
+COPY pyproject.toml .
+COPY poetry.lock .
+RUN pip3 install poetry && poetry install
+
+
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest as deploy
+RUN microdnf -y update && \
+    microdnf -y install \
+        shadow-utils python39 && \
+    microdnf clean all
 
 WORKDIR /caikit
-COPY caikit /caikit
 
-RUN yum -y update && yum -y install git git-lfs && yum clean all && \
-    git lfs install && \
-    pip install 'micropipenv[toml]' && \
-    micropipenv install && \
-    rm -rf ~/.cache && \
-    mkdir -p /opt/models && \
-    adduser -g 0 -u 1001 caikit --home-dir /caikit && \
-    chown -R 1001:0 /caikit /opt/models && \
-    chmod -R g=u /caikit /opt/models
+COPY --from=poetry-builder /tmp/poetry/.venv /caikit/
+
+ENV VIRTUAL_ENV=/caikit
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# This is for the use-cases without kserve
-RUN curl -Lo /usr/local/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
-    chmod +x /usr/local/bin/grpc_health_probe
+RUN groupadd --system caikit --gid 1001 && \
+    adduser --system --uid 1001 --gid 0 --groups caikit \
+    --create-home --home-dir /caikit --shell /sbin/nologin \
+    --comment "Caikit User" caikit
 
-USER 1001
+USER caikit
 
-ENV TRANSFORMERS_CACHE="/tmp/transformers_cache" \
-    RUNTIME_LIBRARY='caikit_nlp' \
-    RUNTIME_LOCAL_MODELS_DIR='/opt/models'
+ENV CONFIG_FILES=/caikit/config/caikit.yml
+VOLUME ["/caikit/config/"]
 
-CMD [ "./start-serving.sh" ]
+CMD ["python",  "-m", "caikit.runtime.grpc_server"]

Makefile

@@ -1,39 +1,26 @@
-TGIS_IMAGE=quay.io/opendatahub/text-generation-inference:stable-bafd218
+CAIKIT_IMAGE=quay.io/opendatahub/caikit-tgis-serving
 
-.PHONY: default
+.PHONY: default refresh-poetry-lock-files
 
 default:
-	podman build -t caikit-tgis-serving:$$(git rev-parse --short HEAD) .
+	podman build \
+		-t $(CAIKIT_IMAGE):dev \
+		-t $(CAIKIT_IMAGE):$$(git rev-parse --short HEAD) \
+		.
 
 
-.PHONY: refresh-piplock-files
-
-refresh-piplock-files:
-	podman run --user root -it \
-		--rm -v $$(pwd)/caikit:/app/caikit:z \
-		$(TGIS_IMAGE) \
+refresh-poetry-lock-files: default
+	podman run --user root -it --rm \
+		--volume $$(pwd):/app:z \
+		--workdir /app  \
+		$(CAIKIT_IMAGE):dev \
 		/bin/bash -c " \
-			cd caikit && \
-			yum -y install git && pip install pipenv && \
-			pipenv lock --pre \
+			pip install poetry && \
+			poetry update \
 		"
-
-
-.PHONY: docker-test
-
-docker-test: default
-	podman run -it --rm \
-		--name caikit-tgis-serving-test-$$(git rev-parse --short HEAD) \
-		--volume $$(pwd)/test:/tmp/test:z --volume $$(pwd)/utils:/tmp/utils:z \
-		caikit-tgis-serving:$$(git rev-parse --short HEAD) \
-		/tmp/test/smoke-test.sh
-
-
-.PHONY: shell
 
 shell: default
 	podman run -it --rm \
 		--name caikit-tgis-serving-test-$$(git rev-parse --short HEAD) \
-		--volume $$(pwd)/test:/tmp/test:z --volume $$(pwd)/utils:/tmp/utils:z \
-		caikit-tgis-serving:$$(git rev-parse --short HEAD) \
+		$(CAIKIT_IMAGE):dev \
 		/bin/bash

demo/kserve/built-tip.md

@@ -1,39 +1,38 @@
 # Bootstrap process (optional)
 
-Caikit-TGIS image always requires a Caikit-formatted model. Below are the instructions from converting a model to a Caikit-formatted one.
+Caikit requires equires a Caikit-formatted model. Below are the instructions from converting a model to a Caikit-formatted one.
 
 1. Clone the model repository (or have the model folder in a directory). In the below example, Bloom-560m model repo is cloned.
-~~~
+
+```bash
 yum -y install git git-lfs
 git lfs install
 git clone https://huggingface.co/bigscience/bloom-560m
-~~~
+```
 
-2. Clone the caikit-nlp repo:
-~~~
-git clone https://github.com/caikit/caikit-nlp.git
-~~~
+2. Create a virtual environment with Python 3.9 and install `caikit-nlp`
 
-3. Create a virtual environment with Python 3.9 and install the caikit-nlp
-~~~
-python3 -m virtualenv -p python3.9 venv
+```bash
+python3 -m venv -p python3.9 venv
 source venv/bin/activate
-python3.9 -m pip install ./caikit-nlp
-~~~
+python3.9 -m pip install git+https://github.com/caikit/caikit-nlp.git
+```
+
+3. (Optional) Clone the `caikit-tgis-serving` repo, if not already available.
 
-4. (Optional) Clone the caikit-tgis-serving repo, if not already available.
-~~~
+```bash
 git clone https://github.com/opendatahub-io/caikit-tgis-serving.git
-~~~
+```
+
+4. Invoke the conversion script located in /utils of caikit-tgis-serving repo.
+
+```bash
+caikit-tgis-serving/utils/convert.py --model-path ./bloom-560m/ --model-save-path ./bloom-560m-caikit
+```
 
-5. Invoke the conversion script located in /utils of caikit-tgis-serving repo.
-~~~
-cp caikit-tgis-serving/utils/convert.py . 
-./convert.py --model-path ./bloom-560m/ --model-save-path ./bloom-560m-caikit
-~~~
+5. Move the model folder (ie. `/bloom-560m-caikit`) into desired storage (ie. S3, MinIO, PVC or other)
+6. Do **not** include the model folder name/directory directly in `InferenceService`, but rather point to the directory where the model folder is located. Let's say the `bloom-560m-caikit` directory is located at: `example-models/llm/models/bloom-560m-caikit/`, then `storageUri` value in the InferenceService CR should look like:
 
-6. Move the model folder (ie. `/bloom-560m-caikit`) into desired storage (ie. S3, MinIO, PVC or other)
-7. Do *not* include the model folder name/directory directly in InferenceService, but rather point to the directory where the model folder is located. Let's say the `bloom-560m-caikit` directory is located at: `example-models/llm/models/bloom-560m-caikit/`, then `storageUri` value in the InferenceService CR should look like:
-~~~
+```bash
 storageUri: s3://example-models/llm/models
-~~~
+```

docs/caikit-isvc.yaml

@@ -9,7 +9,10 @@ metadata:
 spec:
   predictor:
     model:
+      apiVersion: serving.kserve.io/v1alpha2
       modelFormat:
         name: caikit
       runtime: caikit-runtime
-      storageUri: <Path to location with config.yml and artifacts directory>
+      storageUri: proto://path/to/model # single model here
+      # Example, using a pvc:
+      # storageUri: pvc://caikit-pvc/flan-t5-small-caikit/

docs/caikit-servingruntime.yaml

@@ -1,26 +1,73 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: caikit-tgis-config
+data:
+  caikit.yml: |
+    runtime:
+      library: caikit_nlp
+      local_models_dir: /mnt/models/
+      lazy_load_local_models: true
+
+    model_management:
+      finders:
+        default:
+          type: MULTI
+          config:
+            finder_priority:
+              - tgis-auto
+        tgis-auto:
+          type: TGIS-AUTO
+          config:
+            test_connection: true
+      initializers:
+        default:
+          type: LOCAL
+          config:
+            backend_priority:
+              - type: TGIS
+                config:
+                  connection:
+                    hostname: localhost:8033
+---
 apiVersion: serving.kserve.io/v1alpha1
 kind: ServingRuntime
 metadata:
   name: caikit-runtime
 spec:
-  containers:
-  - env:
-    - name: RUNTIME_LOCAL_MODELS_DIR
-      value: /mnt/models
-    # TODO: This will eventually point to the official image
-    image: quay.io/opendatahub/caikit-tgis-serving@sha256:20cbc7b312e0dc63903d4a2c7d70b741135ffea31113af63dd8691da4baae0f9
-    name: kserve-container
-    ports:
-    # Note, KServe only allows a single port, this is the gRPC port. Subject to change in the future
-    - containerPort: 8085
-      name: h2c
-      protocol: TCP
-    resources:
-      requests:
-        cpu: 8
-        memory: 16Gi
   multiModel: false
   supportedModelFormats:
-  # Note: this currently *only* supports caikit format models
-  - autoSelect: true
-    name: caikit
+    # Note: this currently *only* supports caikit format models
+    - autoSelect: true
+      name: caikit
+  containers:
+    - name: kserve-container
+      image: quay.io/opendatahub/text-generation-inference:stable
+      command: ["text-generation-launcher"]
+      args: ["--model-name=/mnt/models/artifacts/"]
+      env:
+        - name: TRANSFORMERS_CACHE
+          value: /tmp/transformers_cache
+      # resources: # configure as required
+      #   requests:
+      #     cpu: 8
+      #     memory: 16Gi
+    - name: transformer-container
+      image: quay.io/opendatahub/caikit-tgis-serving:stable
+      volumeMounts:
+        - name: config-volume
+          mountPath: /caikit/config/
+          readOnly: true
+      ports:
+        - containerPort: 8085
+          name: h2c
+          protocol: TCP
+      # resources: # configure as required
+      #   requests:
+      #     cpu: 8
+      #     memory: 16Gi
+  volumes:
+    - name: config-volume
+      configMap:
+        name: caikit-tgis-config