-
Notifications
You must be signed in to change notification settings - Fork 44
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
4 changed files
with
104 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
/models/ | ||
/.venv/ | ||
/flan-t5-small |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
runtime: | ||
library: caikit_nlp | ||
local_models_dir: /mnt/models | ||
lazy_load_local_models: true | ||
|
||
model_management: | ||
finders: | ||
default: | ||
type: MULTI | ||
config: | ||
finder_priority: | ||
- tgis-auto | ||
# - local | ||
tgis-auto: | ||
type: TGIS-AUTO | ||
config: | ||
test_connections: true | ||
initializers: | ||
default: | ||
type: LOCAL | ||
config: | ||
backend_priority: | ||
- type: TGIS | ||
config: | ||
connection: | ||
hostname: tgis:8033 | ||
test_connections: true | ||
|
||
log: | ||
formatter: pretty |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
services: | ||
caikit: | ||
# image: quay.io/opendatahub/caikit:latest | ||
build: .. | ||
volumes: | ||
- ./caikit_config:/caikit/config | ||
- ./models/flan-t5-small-caikit/:/mnt/models/ | ||
ports: | ||
- 8085:8085 | ||
tgis: | ||
image: quay.io/opendatahub/text-generation-inference:stable | ||
command: [ | ||
"text-generation-launcher", # NOTE:--num-shard defaults to 1 | ||
"--model-name=/mnt/models/artifacts/", | ||
"--max-batch-size=256", | ||
"--max-concurrent-requests=64", | ||
] | ||
volumes: | ||
- ./models/flan-t5-small-caikit/:/mnt/models | ||
environment: | ||
- "TRANSFORMERS_CACHE=/tmp/transformers_cache" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
#!/bin/bash | ||
set -euo pipefail | ||
|
||
function command_required() { | ||
local cmd | ||
cmd=$1 | ||
if ! command -v $cmd &>/dev/null; then | ||
echo "This requires $cmd" 2>&1 | ||
exit 1 | ||
fi | ||
} | ||
|
||
command_required python3 | ||
command_required docker-compose | ||
command_required git-lfs | ||
|
||
if [[ ! -d .venv ]]; then | ||
python -m venv .venv || (echo "Failed to create venv. Quitting." >&2 && exit 1) | ||
fi | ||
source .venv/bin/activate || (echo "Could not source venv. Quitting" >&2 && exit 1) | ||
|
||
pip install git+https://github.com/caikit/caikit-nlp | ||
if [[ ! -d flan-t5-small ]]; then | ||
git clone https://huggingface.co/google/flan-t5-small | ||
fi | ||
|
||
mkdir -p models/ | ||
../utils/convert.py --model-path flan-t5-small --model-save-path models/flan-t5-small-caikit/ | ||
|
||
echo "Saved caikit model to ./models/" | ||
|
||
docker-compose up -d | ||
|
||
max_retries=10 | ||
until grpcurl -plaintext \ | ||
-d '{"text": "At what temperature celsius does Nitrogen boil?"}' \ | ||
-H "mm-model-id: flan-t5-small-caikit" \ | ||
127.0.0.1:8085 \ | ||
caikit.runtime.Nlp.NlpService/TextGenerationTaskPredict; do | ||
sleep 3 | ||
max_retries=$((max_retries - 1)) | ||
if [[ $max_retries -le 0 ]]; then | ||
echo "Failed to query grpc service" >&2 | ||
docker-compose down | ||
exit 1 | ||
fi | ||
done | ||
|
||
docker-compose down | ||
echo "👍 Test successful!" |