From b67d2865c6be4ff444babed90fde8015a3c15289 Mon Sep 17 00:00:00 2001 From: Cathy Zhang Date: Wed, 23 Oct 2024 04:00:06 -0700 Subject: [PATCH] Create new benchmark/performance Signed-off-by: Cathy Zhang --- .../kubernetes/intel/gaudi/README.md | 199 +++++++++++++ .../kubernetes/intel/gaudi/benchmark.sh | 99 +++++++ .../kubernetes/intel/gaudi/benchmark.yaml | 69 +++++ .../kubernetes/intel/gaudi/deploy.py | 277 ++++++++++++++++++ .../intel/gaudi/generate_helm_values.py | 183 ++++++++++++ 5 files changed, 827 insertions(+) create mode 100644 ChatQnA/benchmark/performance/kubernetes/intel/gaudi/README.md create mode 100755 ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.sh create mode 100644 ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.yaml create mode 100644 ChatQnA/benchmark/performance/kubernetes/intel/gaudi/deploy.py create mode 100644 ChatQnA/benchmark/performance/kubernetes/intel/gaudi/generate_helm_values.py diff --git a/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/README.md b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/README.md new file mode 100644 index 000000000..a18602140 --- /dev/null +++ b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/README.md @@ -0,0 +1,199 @@ +# ChatQnA Benchmarking + +This folder contains a collection of Kubernetes manifest files for deploying the ChatQnA service across scalable nodes. It includes a comprehensive [benchmarking tool](https://github.com/opea-project/GenAIEval/blob/main/evals/benchmark/README.md) that enables throughput analysis to assess inference performance. + +By following this guide, you can run benchmarks on your deployment and share the results with the OPEA community. + +## Purpose + +We aim to run these benchmarks and share them with the OPEA community for three primary reasons: + +- To offer insights on inference throughput in real-world scenarios, helping you choose the best service or deployment for your needs. +- To establish a baseline for validating optimization solutions across different implementations, providing clear guidance on which methods are most effective for your use case. +- To inspire the community to build upon our benchmarks, allowing us to better quantify new solutions in conjunction with current leading llms, serving frameworks etc. + +## Metrics + +The benchmark will report the below metrics, including: + +- Number of Concurrent Requests +- End-to-End Latency: P50, P90, P99 (in milliseconds) +- End-to-End First Token Latency: P50, P90, P99 (in milliseconds) +- Average Next Token Latency (in milliseconds) +- Average Token Latency (in milliseconds) +- Requests Per Second (RPS) +- Output Tokens Per Second +- Input Tokens Per Second + +Results will be displayed in the terminal and saved as CSV file named `1_stats.csv` for easy export to spreadsheets. + +## Table of Contents + +- [Deployment](#deployment) + - [Prerequisites](#prerequisites) + - [Deployment Scenarios](#deployment-scenarios) + - [Case 1: Baseline Deployment with Rerank](#case-1-baseline-deployment-with-rerank) + - [Case 2: Baseline Deployment without Rerank](#case-2-baseline-deployment-without-rerank) + - [Case 3: Tuned Deployment with Rerank](#case-3-tuned-deployment-with-rerank) +- [Benchmark](#benchmark) + - [Test Configurations](#test-configurations) + - [Test Steps](#test-steps) + - [Upload Retrieval File](#upload-retrieval-file) + - [Run Benchmark Test](#run-benchmark-test) + - [Data collection](#data-collection) + - [Benchmark multiple nodes](#benchmark-multiple-nodes) +- [Teardown](#teardown) + +## Deployment + +### Prerequisites + +- Kubernetes installation: Use [kubespray](https://github.com/opea-project/docs/blob/main/guide/installation/k8s_install/k8s_install_kubespray.md) or other official Kubernetes installation guides. +- Helm installation: Follow the [Helm documentation](https://helm.sh/docs/intro/install/#helm) to install Helm. +- Setup Hugging Face Token + + To access models and APIs from Hugging Face, set your token as environment variable. + ```bash + export HF_TOKEN="insert-your-huggingface-token-here" + ``` +- Prepare Shared Models (Optional but Strongly Recommended) + + Downloading models simultaneously to multiple nodes in your cluster can overload resources such as network bandwidth, memory and storage. To prevent resource exhaustion, it's recommended to preload the models in advance. + ```bash + pip install -U "huggingface_hub[cli]" + sudo mkdir -p /mnt/models + sudo chmod 777 /mnt/models + huggingface-cli download --cache-dir /mnt/models Intel/neural-chat-7b-v3-3 + export MODEL_DIR=/mnt/models + ``` + Once the models are downloaded, you can consider the following methods for sharing them across nodes: + - Persistent Volume Claim (PVC): This is the recommended approach for production setups. For more details on using PVC, refer to [PVC](https://github.com/opea-project/GenAIInfra/blob/main/helm-charts/README.md#using-persistent-volume). + - Local Host Path: For simpler testing, ensure that each node involved in the deployment follows the steps above to locally prepare the models. After preparing the models, use `--set global.modelUseHostPath=${MODELDIR}` in the deployment command. + +- Add OPEA Helm Repository: + ```bash + python deploy.py --add-repo + ``` +- Label Nodes + ```base + python deploy.py --add-label --num-nodes 2 + ``` + +### Deployment Scenarios + +The example below are based on a two-node setup. You can adjust the number of nodes by using the `--num-nodes` option. + +By default, these commands use the `default` namespace. To specify a different namespace, use the `--namespace` flag with deploy, uninstall, and kubernetes command. Additionally, update the `namespace` field in `benchmark.yaml` before running the benchmark test. + +For additional configuration options, run `python deploy.py --help` + +#### Case 1: Baseline Deployment with Rerank + +Deploy Command (with node number, Hugging Face token, model directory specified): +```bash +python deploy.py --hf-token $HF_TOKEN --model-dir $MODEL_DIR --num-nodes 2 --with-rerank +``` +Uninstall Command: +```bash +python deploy.py --uninstall +``` + +#### Case 2: Baseline Deployment without Rerank + +```bash +python deploy.py --hftoken $HFTOKEN --modeldir $MODELDIR --num-nodes 2 +``` +#### Case 3: Tuned Deployment with Rerank + +```bash +python deploy.py --hftoken $HFTOKEN --modeldir $MODELDIR --num-nodes 2 --with-rerank --tuned +``` + +## Benchmark + +### Test Configurations + +| Key | Value | +| -------- | ------- | +| Workload | ChatQnA | +| Tag | V1.1 | + +Models configuration +| Key | Value | +| ---------- | ------------------ | +| Embedding | BAAI/bge-base-en-v1.5 | +| Reranking | BAAI/bge-reranker-base | +| Inference | Intel/neural-chat-7b-v3-3 | + +Benchmark parameters +| Key | Value | +| ---------- | ------------------ | +| LLM input tokens | 1024 | +| LLM output tokens | 128 | + +Number of test requests for different scheduled node number: +| Node count | Concurrency | Query number | +| ----- | -------- | -------- | +| 1 | 128 | 640 | +| 2 | 256 | 1280 | +| 4 | 512 | 2560 | + +More detailed configuration can be found in configuration file [benchmark.yaml](./benchmark.yaml). + +### Test Steps + +Use `kubectl get pods` to confirm that all pods are `Ready` before starting the test. + +#### Upload Retrieval File + +Before testing, upload a specified file to make sure the llm input have the token length of 1k. + +Retrieve the `ClusterIP` of the `chatqna-data-prep` service. + +```bash +kubectl get svc +``` +Expected output: +```log +chatqna-data-prep ClusterIP xx.xx.xx.xx 6007/TCP 51m +``` + +Use the following `cURL` command to upload file: + +```bash +cd GenAIEval/evals/benchmark/data +# RAG with Rerank +curl -X POST "http://${cluster_ip}:6007/v1/dataprep" \ + -H "Content-Type: multipart/form-data" \ + -F "files=@./GenAIEval/evals/benchmark/data/upload_file.txt" \ + -F "chunk_size=3800" +# RAG without Rerank +curl -X POST "http://${cluster_ip}:6007/v1/dataprep" \ + -H "Content-Type: multipart/form-data" \ + -F "files=@./GenAIEval/evals/benchmark/data/upload_file_no_rerank.txt" +``` + +#### Run Benchmark Test + +Run the benchmark test using: +```bash +bash benchmark.sh -n 2 +``` +The `-n` argument specifies the number of test nodes. Required dependencies will be automatically installed when running the benchmark for the first time. + +#### Data collection + +All the test results will come to the folder `GenAIEval/evals/benchmark/benchmark_output`. + +## Teardown + +After completing the benchmark, use the following commands to clean up the environment: + +Remove Node Labels: +```base +python deploy.py --delete-label +``` +Delete the OPEA Helm Repository: +```bash +python deploy.py --delete-repo +``` diff --git a/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.sh b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.sh new file mode 100755 index 000000000..ba69f4e96 --- /dev/null +++ b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +deployment_type="k8s" +node_number=1 +service_port=8888 +query_per_node=640 + +benchmark_tool_path="$(pwd)/GenAIEval" + +usage() { + echo "Usage: $0 [-d deployment_type] [-n node_number] [-i service_ip] [-p service_port]" + echo " -d deployment_type ChatQnA deployment type, select between k8s and docker (default: k8s)" + echo " -n node_number Test node number, required only for k8s deployment_type, (default: 1)" + echo " -i service_ip chatqna service ip, required only for docker deployment_type" + echo " -p service_port chatqna service port, required only for docker deployment_type, (default: 8888)" + exit 1 +} + +while getopts ":d:n:i:p:" opt; do + case ${opt} in + d ) + deployment_type=$OPTARG + ;; + n ) + node_number=$OPTARG + ;; + i ) + service_ip=$OPTARG + ;; + p ) + service_port=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" 1>&2 + usage + ;; + : ) + echo "Invalid option: -$OPTARG requires an argument" 1>&2 + usage + ;; + esac +done + +if [[ "$deployment_type" == "docker" && -z "$service_ip" ]]; then + echo "Error: service_ip is required for docker deployment_type" 1>&2 + usage +fi + +if [[ "$deployment_type" == "k8s" && ( -n "$service_ip" || -n "$service_port" ) ]]; then + echo "Warning: service_ip and service_port are ignored for k8s deployment_type" 1>&2 +fi + +function main() { + if [[ ! -d ${benchmark_tool_path} ]]; then + echo "Benchmark tool not found, setting up..." + setup_env + fi + run_benchmark +} + +function setup_env() { + git clone https://github.com/opea-project/GenAIEval.git + pushd ${benchmark_tool_path} + python3 -m venv stress_venv + source stress_venv/bin/activate + pip install -r requirements.txt + popd +} + +function run_benchmark() { + source ${benchmark_tool_path}/stress_venv/bin/activate + export DEPLOYMENT_TYPE=${deployment_type} + export SERVICE_IP=${service_ip:-"None"} + export SERVICE_PORT=${service_port:-"None"} + if [[ -z $USER_QUERIES ]]; then + user_query=$((query_per_node*node_number)) + export USER_QUERIES="[${user_query}, ${user_query}, ${user_query}, ${user_query}]" + echo "USER_QUERIES not configured, setting to: ${USER_QUERIES}." + fi + export WARMUP=$(echo $USER_QUERIES | sed -e 's/[][]//g' -e 's/,.*//') + if [[ -z $WARMUP ]]; then export WARMUP=0; fi + if [[ -z $TEST_OUTPUT_DIR ]]; then + if [[ $DEPLOYMENT_TYPE == "k8s" ]]; then + export TEST_OUTPUT_DIR="${benchmark_tool_path}/evals/benchmark/benchmark_output/node_${node_number}" + else + export TEST_OUTPUT_DIR="${benchmark_tool_path}/evals/benchmark/benchmark_output/docker" + fi + echo "TEST_OUTPUT_DIR not configured, setting to: ${TEST_OUTPUT_DIR}." + fi + + envsubst < ./benchmark.yaml > ${benchmark_tool_path}/evals/benchmark/benchmark.yaml + cd ${benchmark_tool_path}/evals/benchmark + python benchmark.py +} + +main diff --git a/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.yaml b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.yaml new file mode 100644 index 000000000..1d4ae4794 --- /dev/null +++ b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.yaml @@ -0,0 +1,69 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +test_suite_config: # Overall configuration settings for the test suite + examples: ["chatqna"] # The specific test cases being tested, e.g., chatqna, codegen, codetrans, faqgen, audioqna, visualqna + deployment_type: ${DEPLOYMENT_TYPE} # Default is "k8s", can also be "docker" + service_ip: ${SERVICE_IP} # Leave as None for k8s, specify for Docker + service_port: ${SERVICE_PORT} # Leave as None for k8s, specify for Docker + warm_ups: ${WARMUP} # Number of test requests for warm-up + run_time: 60m # The max total run time for the test suite + seed: # The seed for all RNGs + user_queries: ${USER_QUERIES} # Number of test requests at each concurrency level + query_timeout: 120 # Number of seconds to wait for a simulated user to complete any executing task before exiting. 120 sec by defeult. + random_prompt: false # Use random prompts if true, fixed prompts if false + collect_service_metric: false # Collect service metrics if true, do not collect service metrics if false + data_visualization: false # Generate data visualization if true, do not generate data visualization if false + llm_model: "Intel/neural-chat-7b-v3-3" # The LLM model used for the test + test_output_dir: "${TEST_OUTPUT_DIR}" # The directory to store the test output + load_shape: # Tenant concurrency pattern + name: constant # poisson or constant(locust default load shape) + params: # Loadshape-specific parameters + constant: # Constant load shape specific parameters, activate only if load_shape.name is constant + concurrent_level: 5 # If user_queries is specified, concurrent_level is target number of requests per user. If not, it is the number of simulated users + # arrival_rate: 1.0 # Request arrival rate. If set, concurrent_level will be overridden, constant load will be generated based on arrival-rate + poisson: # Poisson load shape specific parameters, activate only if load_shape.name is poisson + arrival_rate: 1.0 # Request arrival rate + namespace: "my-chatqna" + +test_cases: + chatqna: + embedding: + run_test: false + service_name: "chatqna-embedding-usvc" # Replace with your service name + embedserve: + run_test: false + service_name: "chatqna-tei" # Replace with your service name + retriever: + run_test: false + service_name: "chatqna-retriever-usvc" # Replace with your service name + parameters: + search_type: "similarity" + k: 4 + fetch_k: 20 + lambda_mult: 0.5 + score_threshold: 0.2 + reranking: + run_test: false + service_name: "chatqna-reranking-usvc" # Replace with your service name + parameters: + top_n: 1 + rerankserve: + run_test: false + service_name: "chatqna-teirerank" # Replace with your service name + llm: + run_test: false + service_name: "chatqna-llm-uservice" # Replace with your service name + parameters: + max_tokens: 128 + temperature: 0.01 + top_k: 10 + top_p: 0.95 + repetition_penalty: 1.03 + streaming: true + llmserve: + run_test: false + service_name: "chatqna-tgi" # Replace with your service name + e2e: + run_test: true + service_name: "chatqna" # Replace with your service name diff --git a/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/deploy.py b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/deploy.py new file mode 100644 index 000000000..817ba0294 --- /dev/null +++ b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/deploy.py @@ -0,0 +1,277 @@ +import os +import subprocess +import json +import argparse +import yaml +from generate_helm_values import generate_helm_values + +def run_kubectl_command(command): + """Run a kubectl command and return the output.""" + try: + result = subprocess.run(command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return result.stdout + except subprocess.CalledProcessError as e: + print(f"Error running command: {command}\n{e.stderr}") + exit(1) + +def get_all_nodes(): + """Get the list of all nodes in the Kubernetes cluster.""" + command = ["kubectl", "get", "nodes", "-o", "json"] + output = run_kubectl_command(command) + nodes = json.loads(output) + return [node['metadata']['name'] for node in nodes['items']] + +def add_label_to_node(node_name, label): + """Add a label to the specified node.""" + command = ["kubectl", "label", "node", node_name, label, "--overwrite"] + print(f"Labeling node {node_name} with {label}...") + run_kubectl_command(command) + print(f"Label {label} added to node {node_name} successfully.") + +def add_labels_to_nodes(node_count=None, label=None, node_names=None): + """Add a label to the specified number of nodes or to specified nodes.""" + + if node_names: + # Add label to the specified nodes + for node_name in node_names: + add_label_to_node(node_name, label) + else: + # Fetch the node list and label the specified number of nodes + all_nodes = get_all_nodes() + if node_count is None or node_count > len(all_nodes): + print(f"Error: Node count exceeds the number of available nodes ({len(all_nodes)} available).") + sys.exit(1) + + selected_nodes = all_nodes[:node_count] + for node_name in selected_nodes: + add_label_to_node(node_name, label) + +def clear_labels_from_nodes(label, node_names=None): + """Clear the specified label from specific nodes if provided, otherwise from all nodes.""" + label_key = label.split('=')[0] # Extract key from 'key=value' format + + # If specific nodes are provided, use them; otherwise, get all nodes + nodes_to_clear = node_names if node_names else get_all_nodes() + + for node_name in nodes_to_clear: + # Check if the node has the label by inspecting its metadata + command = ["kubectl", "get", "node", node_name, "-o", "json"] + node_info = run_kubectl_command(command) + node_metadata = json.loads(node_info) + + # Check if the label exists on this node + labels = node_metadata['metadata'].get('labels', {}) + if label_key in labels: + # Remove the label from the node + command = ["kubectl", "label", "node", node_name, f"{label_key}-"] + print(f"Removing label {label_key} from node {node_name}...") + run_kubectl_command(command) + print(f"Label {label_key} removed from node {node_name} successfully.") + else: + print(f"Label {label_key} not found on node {node_name}, skipping.") + +def add_helm_repo(repo_name, repo_url): + # Add the repo if it does not exist + add_command = ["helm", "repo", "add", repo_name, repo_url] + try: + subprocess.run(add_command, check=True) + print(f"Added Helm repo {repo_name} from {repo_url}.") + except subprocess.CalledProcessError as e: + print(f"Failed to add Helm repo {repo_name}: {e}") + +def delete_helm_repo(repo_name): + """Delete Helm repo if it exists.""" + command = ["helm", "repo", "remove", repo_name] + try: + subprocess.run(command, check=True) + print(f"Deleted Helm repo {repo_name}.") + except subprocess.CalledProcessError: + print(f"Failed to delete Helm repo {repo_name}. It may not exist.") + +def configmap_exists(name, namespace): + """Check if a ConfigMap exists in the specified namespace.""" + check_command = ["kubectl", "get", "configmap", name, "-n", namespace] + result = subprocess.run( + check_command, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + return result.returncode == 0 + +def create_configmap(name, namespace, data): + """Create a ConfigMap if it does not already exist.""" + if configmap_exists(name, namespace): + print(f"ConfigMap '{name}' already exists in namespace '{namespace}', skipping creation.") + else: + create_command = ["kubectl", "create", "configmap", name] + [ + f"--from-literal={k}={v}" for k, v in data.items() + ] + ["-n", namespace] + print(f"Creating ConfigMap '{name}' in namespace '{namespace}'...") + subprocess.run(create_command, check=True) + print(f"ConfigMap '{name}' created successfully.") + +def delete_configmap(name, namespace): + """Delete a ConfigMap if it exists.""" + if configmap_exists(name, namespace): + delete_command = ["kubectl", "delete", "configmap", name, "-n", namespace] + print(f"Deleting ConfigMap '{name}'...") + subprocess.run(delete_command, check=True) + print(f"ConfigMap '{name}' deleted successfully.") + else: + print(f"ConfigMap '{name}' does not exist in namespace '{namespace}', skipping deletion.") + +def install_helm_release(release_name, chart_name, namespace, values_file): + """Deploy a Helm release with specified name and chart.""" + + # Check if the namespace exists; if not, create it + try: + # Check if the namespace exists + command = ["kubectl", "get", "namespace", namespace] + subprocess.run(command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + # Namespace does not exist, create it + print(f"Namespace '{namespace}' does not exist. Creating it...") + command = ["kubectl", "create", "namespace", namespace] + subprocess.run(command, check=True) + print(f"Namespace '{namespace}' created successfully.") + + # This is workaround for teirerank-gaudi, will be removed later + create_configmap("extra-env", namespace, {"MAX_WARMUP_SEQUENCE_LENGTH": "512"}) + + # Prepare the Helm install command + command = [ + "helm", "install", release_name, chart_name, + "--namespace", namespace, "-f", values_file + ] + + # Execute the Helm install command + try: + print(f"Running command: {' '.join(command)}") # Print full command for debugging + subprocess.run(command, check=True) + print("Deployment initiated successfully.") + except subprocess.CalledProcessError as e: + print(f"Error occurred while deploying Helm release: {e}") + +def uninstall_helm_release(release_name, namespace=None): + """Uninstall a Helm release and clean up resources, optionally delete the namespace if not 'default'.""" + # Default to 'default' namespace if none is specified + if not namespace: + namespace = "default" + + try: + # This is workaround for teirerank-gaudi, will be removed later + delete_configmap("extra-env", namespace) + + # Uninstall the Helm release + command = ["helm", "uninstall", release_name, "--namespace", namespace] + print(f"Uninstalling Helm release {release_name} in namespace {namespace}...") + run_kubectl_command(command) + print(f"Helm release {release_name} uninstalled successfully.") + + # If the namespace is specified and not 'default', delete it + if namespace != "default": + print(f"Deleting namespace {namespace}...") + delete_namespace_command = ["kubectl", "delete", "namespace", namespace] + run_kubectl_command(delete_namespace_command) + print(f"Namespace {namespace} deleted successfully.") + else: + print("Namespace is 'default', skipping deletion.") + + except subprocess.CalledProcessError as e: + print(f"Error occurred while uninstalling Helm release or deleting namespace: {e}") + +def main(): + parser = argparse.ArgumentParser(description="Manage Helm Deployment.") + parser.add_argument("--release-name", type=str, default="chatqna", + help="The Helm release name created during deployment (default: chatqna).") + parser.add_argument("--chart-name", type=str, default="opea/chatqna", + help="The chart name to deploy (default: opea/chatqna).") + parser.add_argument("--namespace", default="default", + help="Kubernetes namespace (default: default).") + parser.add_argument("--hf-token", + help="Hugging Face API token.") + parser.add_argument("--model-dir", + help="Model directory, mounted as volumes for service access to pre-downloaded models") + parser.add_argument("--repo-name", default="opea", + help="Helm repo name to add/delete (default: opea).") + parser.add_argument("--repo-url", default="https://opea-project.github.io/GenAIInfra", + help="Helm repository URL (default: https://opea-project.github.io/GenAIInfra).") + parser.add_argument("--user-values", + help="Path to a user-specified values.yaml file.") + parser.add_argument("--create-values-only", action="store_true", + help="Only create the values.yaml file without deploying.") + parser.add_argument("--uninstall", action="store_true", + help="Uninstall the Helm release.") + parser.add_argument("--num-nodes", type=int, default=1, + help="Number of nodes to use (default: 1).") + parser.add_argument("--node-names", nargs='*', + help="Optional specific node names to label.") + parser.add_argument("--add-label", action="store_true", + help="Add label to specified nodes if this flag is set.") + parser.add_argument("--delete-label", action="store_true", + help="Delete label from specified nodes if this flag is set.") + parser.add_argument("--label", default="node-type=opea-benchmark", + help="Label to add/delete (default: node-type=opea-benchmark).") + parser.add_argument("--with-rerank", action="store_true", + help="Include rerank service in the deployment.") + parser.add_argument("--tuned", action="store_true", + help="Modify resources for services and change extraCmdArgs when creating values.yaml.") + parser.add_argument("--add-repo", action="store_true", + help="Add the Helm repo specified by --repo-url.") + parser.add_argument("--delete-repo", action="store_true", + help="Delete the Helm repo specified by --repo-name.") + + args = parser.parse_args() + + # Adjust num-nodes based on node-names if specified + if args.node_names: + num_node_names = len(args.node_names) + if args.num_nodes != 1 and args.num_nodes != num_node_names: + parser.error("--num-nodes must match the number of --node-names if both are specified.") + else: + args.num_nodes = num_node_names + + # Helm repository management + if args.add_repo: + add_helm_repo(args.repo_name, args.repo_url) + return + elif args.delete_repo: + delete_helm_repo(args.repo_name) + return + + # Node labeling management + if args.add_label: + add_labels_to_nodes(args.num_nodes, args.label, args.node_names) + return + elif args.delete_label: + clear_labels_from_nodes(args.label, args.node_names) + return + + # Uninstall Helm release if specified + if args.uninstall: + uninstall_helm_release(args.release_name, args.namespace) + return + + # Prepare values.yaml if not uninstalling + if args.user_values: + values_file_path = args.user_values + else: + if not args.hf_token: + parser.error("--hf-token are required") + node_selector = {args.label.split('=')[0]: args.label.split('=')[1]} + values_file_path = generate_helm_values( + with_rerank=args.with_rerank, + num_nodes=args.num_nodes, + hf_token=args.hf_token, + model_dir=args.model_dir, + node_selector=node_selector, + tune=args.tuned + ) + + # Deploy unless --create-values-only is specified + if not args.create_values_only: + install_helm_release(args.release_name, args.chart_name, args.namespace, values_file_path) + +if __name__ == "__main__": + main() diff --git a/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/generate_helm_values.py b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/generate_helm_values.py new file mode 100644 index 000000000..f5f2ac5a2 --- /dev/null +++ b/ChatQnA/benchmark/performance/kubernetes/intel/gaudi/generate_helm_values.py @@ -0,0 +1,183 @@ +import os +import yaml + +def generate_helm_values(with_rerank, num_nodes, hf_token, model_dir, node_selector=None, tune=False): + """Create a values.yaml file based on the provided configuration.""" + + # Log the received parameters + print("Received parameters:") + print(f"with_rerank: {with_rerank}") + print(f"num_nodes: {num_nodes}") + print(f"node_selector: {node_selector}") # Log the node_selector + print(f"tune: {tune}") + + if node_selector is None: + node_selector = {} + + # Construct the base values dictionary + values = { + "tei": { + "accelDevice": "gaudi", # Default accelDevice + "image": { + "repository": "ghcr.io/huggingface/tei-gaudi", # Default repository + "tag": "latest" # Default tag + }, + "nodeSelector": {key: value for key, value in node_selector.items()} + }, + "tgi": { + "accelDevice": "gaudi", # Default accelDevice + "image": { + "repository": "ghcr.io/huggingface/tgi-gaudi", # Default repository + "tag": "2.0.5" # Default tag + }, + "nodeSelector": {key: value for key, value in node_selector.items()} + }, + "data-prep": { + "nodeSelector": {key: value for key, value in node_selector.items()} + }, + "redis-vector-db": { + "nodeSelector": {key: value for key, value in node_selector.items()} + }, + "retriever-usvc": { + "nodeSelector": {key: value for key, value in node_selector.items()} + }, + "llm-uservice": { + "nodeSelector": {key: value for key, value in node_selector.items()} + }, + "embedding-usvc": { + "nodeSelector": {key: value for key, value in node_selector.items()} + }, + "chatqna-ui": { + "nodeSelector": {key: value for key, value in node_selector.items()} + }, + "global": { + "HUGGINGFACEHUB_API_TOKEN": hf_token, # Use passed token + "modelUseHostPath": model_dir, # Use passed model directory + "extraEnvConfig": "extra-env" # Added MAX_WARMUP_SEQUENCE_LENGTH: 512 to extra-env in deploy.py + }, + "nodeSelector": {key: value for key, value in node_selector.items()} + } + + values["tgi"]["extraCmdArgs"] = ["--max-input-length", "1024", "--max-total-tokens", "2048"] + + if with_rerank: + values["teirerank"] = { + "accelDevice": "gaudi", # Default accelDevice + "image": { + "repository": "ghcr.io/huggingface/tei-gaudi", # Default repository + "tag": "latest" # Default tag + }, + "nodeSelector": {key: value for key, value in node_selector.items()} + } + + values["reranking-usvc"] = { + "nodeSelector": {key: value for key, value in node_selector.items()} + } + else: + values["image"] = { + "repository": "opea/chatqna-without-rerank", + "pullPolicy": "IfNotPresent", + "tag": "latest" + } + + default_replicas = [ + {"name": "chatqna", "replicaCount": 1}, + {"name": "tei", "replicaCount": 1}, + {"name": "teirerank", "replicaCount": 1} if with_rerank else None, + {"name": "tgi", "replicaCount": 6 if with_rerank else 7}, + {"name": "data-prep", "replicaCount": 1}, + {"name": "redis-vector-db", "replicaCount": 1}, + {"name": "retriever", "replicaCount": 2}, + ] + + if num_nodes > 1: + # Scale replicas based on number of nodes + replicas = [ + {"name": "chatqna", "replicaCount": 1 * num_nodes}, + {"name": "tei", "replicaCount": 1 * num_nodes}, + {"name": "teirerank", "replicaCount": 1} if with_rerank else None, + {"name": "tgi", "replicaCount": (8 * num_nodes - 2) if with_rerank else (8 * num_nodes - 1)}, + {"name": "data-prep", "replicaCount": 1}, + {"name": "redis-vector-db", "replicaCount": 1}, + {"name": "retriever", "replicaCount": 1 * num_nodes}, + ] + else: + replicas = default_replicas + + # Remove None values for rerank disabled + replicas = [r for r in replicas if r] + + # Update values.yaml with replicas + for replica in replicas: + service_name = replica["name"] + if service_name == "chatqna": + values["replicaCount"] = replica["replicaCount"] + print(replica["replicaCount"]) + elif service_name in values: + values[service_name]["replicaCount"] = replica["replicaCount"] + + # Prepare resource configurations based on tuning + resources = [] + if tune: + resources = [ + {"name": "chatqna", "resources": {"limits": {"cpu": "16", "memory": "8000Mi"}, "requests": {"cpu": "16", "memory": "8000Mi"}}}, + {"name": "tei", "resources": {"limits": {"cpu": "80", "memory": "20000Mi"}, "requests": {"cpu": "80", "memory": "20000Mi"}}}, + {"name": "teirerank", "resources": {"limits": {"habana.ai/gaudi": 1}}} if with_rerank else None, + {"name": "tgi", "resources": {"limits": {"habana.ai/gaudi": 1}}}, + {"name": "retriever", "resources": {"requests": {"cpu": "8", "memory": "8000Mi"}}}, + ] + + # Filter out any None values directly as part of initialization + resources = [r for r in resources if r is not None] + + # Add resources for each service if tuning + for resource in resources: + service_name = resource["name"] + if service_name == "chatqna": + values["resources"] = resource["resources"] + elif service_name in values: + values[service_name]["resources"] = resource["resources"] + + # Add extraCmdArgs for tgi service with default values + if "tgi" in values: + values["tgi"]["extraCmdArgs"] = ["--max-input-length", "1280", "--max-total-tokens", "2048", + "--max-batch-total-tokens", "65536", "--max-batch-prefill-tokens", "4096"] + + yaml_string = yaml.dump(values, default_flow_style=False) + + # Determine the mode based on the 'tune' parameter + mode = "tuned" if tune else "oob" + + # Determine the filename based on 'with_rerank' and 'num_nodes' + if with_rerank: + filename = f"{mode}_{num_nodes}_gaudi_with_rerank.yaml" + else: + filename = f"{mode}_{num_nodes}_gaudi_without_rerank.yaml" + + # Write the YAML data to the file + with open(filename, "w") as file: + file.write(yaml_string) + + # Get the current working directory and construct the file path + current_dir = os.getcwd() + filepath = os.path.join(current_dir, filename) + + print(f"YAML file {filepath} has been generated.") + return filepath # Optionally return the file path + +# Main execution for standalone use of create_values_yaml +if __name__ == "__main__": + # Example values for standalone execution + with_rerank = True + num_nodes = 2 + hftoken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + modeldir = "/mnt/model" + node_selector = {"node-type": "opea-benchmark"} + tune = True + + filename = generate_helm_values(with_rerank, num_nodes, hftoken, modeldir, node_selector, tune) + + # Read back the generated YAML file for verification + with open(filename, 'r') as file: + print("Generated YAML contents:") + print(file.read())