diff --git a/charts/kubeai/templates/role.yaml b/charts/kubeai/templates/role.yaml index 6e393870..51c4f356 100644 --- a/charts/kubeai/templates/role.yaml +++ b/charts/kubeai/templates/role.yaml @@ -9,10 +9,10 @@ rules: - "" resources: - pods - - persistentvolumeclaims verbs: - create - delete + - deletecollection - get - list - patch @@ -25,6 +25,19 @@ rules: verbs: - create - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - create + - delete - get - list - patch diff --git a/docs/how-to/cache-models-with-gcp-filestore.md b/docs/how-to/cache-models-with-gcp-filestore.md index 805daced..4ef16099 100644 --- a/docs/how-to/cache-models-with-gcp-filestore.md +++ b/docs/how-to/cache-models-with-gcp-filestore.md @@ -15,21 +15,34 @@ gcloud services enable file.googleapis.com Apply a Model with the cache profile set to `standard-filestore` (defined in the reference [GKE Helm values file](https://github.com/substratusai/kubeai/blob/main/charts/kubeai/values-gke.yaml)). +
+TIP: If you want to use `premium-filestore` you will need to ensure you have quota. +Open the cloud console quotas page: https://console.cloud.google.com/iam-admin/quotas. Make sure your project is selected in the top left. + +Ensure that you have at least 2.5Tb of `PremiumStorageGbPerRegion` quota in the region where your cluster is deployed. + +![Premium Storage Quota Screenshot](../screenshots/gcp-quota-premium-storage-gb-per-region) + +
+
+ NOTE: If you already installed the models chart, you will need to edit you values file and run `helm upgrade`. ```bash -helm install kubeai-models $REPO_DIR/charts/models -f - < +Example: Out-of-quota error +``` + Warning ProvisioningFailed 11m (x26 over 21m) filestore.csi.storage.gke.io_gke-50826743a27a4d52bf5b-7fac-9607-vm_b4bdb2ec-b58b-4363-adec-15c270a14066 failed to provision volume with StorageClass "premium-rwx": rpc error: code = ResourceExhausted desc = googleapi: Error 429: Quota limit 'PremiumStorageGbPerRegion' has been exceeded. Limit: 0 in region us-central1. +Details: +[ + { + "@type": "type.googleapis.com/google.rpc.QuotaFailure", + "violations": [ + { + "description": "Quota 'PremiumStorageGbPerRegion' exhausted. Limit 0 in region us-central1", + "subject": "project:819220466562" + } + ] + } +] +``` + + +Check to see if the PersistentVolume has been fully provisioned. + +```bash +kubectl get pv +# Find name of corresponding pv... +kubectl describe pv +``` + ### Model Loading Job Check to see if there is an ongoing model loader Job. diff --git a/docs/screenshots/gcp-quota-premium-storage-gb-per-region.png b/docs/screenshots/gcp-quota-premium-storage-gb-per-region.png new file mode 100644 index 00000000..1e78ab9e Binary files /dev/null and b/docs/screenshots/gcp-quota-premium-storage-gb-per-region.png differ diff --git a/skaffold.yaml b/skaffold.yaml index 646b11a6..5335cec3 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -43,8 +43,8 @@ profiles: - name: kubeai-only-gke build: - artifacts: - - image: substratusai/kubeai + local: + push: true deploy: helm: releases: