From e7f1d099079e0806317bfc91305009c77f81c210 Mon Sep 17 00:00:00 2001 From: Cory Latschkowski Date: Thu, 3 Oct 2024 17:50:15 -0500 Subject: [PATCH 1/4] update: remove default taint --- .../overlays/toleration/kustomization.yaml | 6 ------ .../instance/overlays/nvidia-gpu/kustomization.yaml | 3 --- .../instance/base/cluster-policy.yaml | 3 --- .../instance/components/aws-gpu-machineset/ocp.sh | 10 +++------- .../time-sliced-4/patch-device-plugin-config.yaml | 5 ----- .../time-sliced/patch-device-plugin-config.yaml | 5 ----- .../nvidia-profile.yaml | 3 --- scripts/library/ocp.sh | 12 ++++-------- .../parasol-insurance/01-setup/kustomization.yaml | 3 --- 9 files changed, 7 insertions(+), 43 deletions(-) diff --git a/components/app-configs/nvidia-gpu-verification/overlays/toleration/kustomization.yaml b/components/app-configs/nvidia-gpu-verification/overlays/toleration/kustomization.yaml index 57f908b6..b196a3b0 100644 --- a/components/app-configs/nvidia-gpu-verification/overlays/toleration/kustomization.yaml +++ b/components/app-configs/nvidia-gpu-verification/overlays/toleration/kustomization.yaml @@ -16,9 +16,6 @@ patches: - effect: NoSchedule key: nvidia.com/gpu operator: Exists - - effect: NoSchedule - key: nvidia-gpu-only - operator: Exists - target: version: v1 kind: Pod @@ -29,6 +26,3 @@ patches: - effect: NoSchedule key: nvidia.com/gpu operator: Exists - - effect: NoSchedule - key: nvidia-gpu-only - operator: Exists diff --git a/components/operators/devspaces/instance/overlays/nvidia-gpu/kustomization.yaml b/components/operators/devspaces/instance/overlays/nvidia-gpu/kustomization.yaml index f01c1227..06c6f564 100644 --- a/components/operators/devspaces/instance/overlays/nvidia-gpu/kustomization.yaml +++ b/components/operators/devspaces/instance/overlays/nvidia-gpu/kustomization.yaml @@ -28,6 +28,3 @@ patches: - effect: NoSchedule key: nvidia.com/gpu operator: Exists - - effect: NoSchedule - key: nvidia-gpu-only - operator: Exists diff --git a/components/operators/gpu-operator-certified/instance/base/cluster-policy.yaml b/components/operators/gpu-operator-certified/instance/base/cluster-policy.yaml index b321c827..1558e29e 100644 --- a/components/operators/gpu-operator-certified/instance/base/cluster-policy.yaml +++ b/components/operators/gpu-operator-certified/instance/base/cluster-policy.yaml @@ -55,9 +55,6 @@ spec: - effect: NoSchedule key: nvidia.com/gpu operator: Exists - - effect: NoSchedule - key: nvidia-gpu-only - operator: Exists devicePlugin: enabled: true config: diff --git a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh index f6f1389f..7209e811 100644 --- a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh +++ b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh @@ -46,15 +46,11 @@ ocp_aws_create_gpu_machineset(){ patch "${MACHINE_SET_TYPE}" \ --type=merge --patch '{"spec":{"template":{"spec":{"metadata":{"labels":{"node-role.kubernetes.io/gpu":""}}}}}}' - # taint nodes for gpu-only workloads - oc -n openshift-machine-api \ - patch "${MACHINE_SET_TYPE}" \ - --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia.com/gpu","value":"","effect":"NoSchedule"}]}}}}' - + # # taint nodes for gpu-only workloads # oc -n openshift-machine-api \ # patch "${MACHINE_SET_TYPE}" \ - # --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia-gpu-only","value":"","effect":"NoSchedule"}]}}}}' - + # --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia.com/gpu","value":"","effect":"NoSchedule"}]}}}}' + # should use the default profile # oc -n openshift-machine-api \ # patch "${MACHINE_SET_TYPE}" \ diff --git a/components/operators/gpu-operator-certified/instance/components/time-sliced-4/patch-device-plugin-config.yaml b/components/operators/gpu-operator-certified/instance/components/time-sliced-4/patch-device-plugin-config.yaml index 67d29451..1f3e9273 100644 --- a/components/operators/gpu-operator-certified/instance/components/time-sliced-4/patch-device-plugin-config.yaml +++ b/components/operators/gpu-operator-certified/instance/components/time-sliced-4/patch-device-plugin-config.yaml @@ -5,11 +5,6 @@ metadata: data: no-time-sliced: |- version: v1 - sharing: - timeSlicing: - resources: - - name: nvidia.com/gpu - replicas: 0 time-sliced: |- version: v1 sharing: diff --git a/components/operators/gpu-operator-certified/instance/components/time-sliced/patch-device-plugin-config.yaml b/components/operators/gpu-operator-certified/instance/components/time-sliced/patch-device-plugin-config.yaml index fffd36d3..40bcca6f 100644 --- a/components/operators/gpu-operator-certified/instance/components/time-sliced/patch-device-plugin-config.yaml +++ b/components/operators/gpu-operator-certified/instance/components/time-sliced/patch-device-plugin-config.yaml @@ -5,11 +5,6 @@ metadata: data: no-time-sliced: |- version: v1 - sharing: - timeSlicing: - resources: - - name: nvidia.com/gpu - replicas: 0 time-sliced: |- version: v1 sharing: diff --git a/components/operators/rhods-operator/instance/components/nvidia-gpu-accelerator-profile/nvidia-profile.yaml b/components/operators/rhods-operator/instance/components/nvidia-gpu-accelerator-profile/nvidia-profile.yaml index f35657e2..62112f00 100644 --- a/components/operators/rhods-operator/instance/components/nvidia-gpu-accelerator-profile/nvidia-profile.yaml +++ b/components/operators/rhods-operator/instance/components/nvidia-gpu-accelerator-profile/nvidia-profile.yaml @@ -14,6 +14,3 @@ spec: - effect: NoSchedule key: nvidia.com/gpu operator: Exists - - effect: NoSchedule - key: nvidia-gpu-only - operator: Exists diff --git a/scripts/library/ocp.sh b/scripts/library/ocp.sh index 78a80cea..06e84975 100644 --- a/scripts/library/ocp.sh +++ b/scripts/library/ocp.sh @@ -244,14 +244,10 @@ ocp_aws_create_gpu_machineset(){ patch "${MACHINE_SET_TYPE}" \ --type=merge --patch '{"spec":{"template":{"spec":{"metadata":{"labels":{"node-role.kubernetes.io/gpu":""}}}}}}' - # taint nodes for gpu-only workloads - oc -n openshift-machine-api \ - patch "${MACHINE_SET_TYPE}" \ - --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia.com/gpu","value":"","effect":"NoSchedule"}]}}}}' - + # # taint nodes for gpu-only workloads # oc -n openshift-machine-api \ # patch "${MACHINE_SET_TYPE}" \ - # --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia-gpu-only","value":"","effect":"NoSchedule"}]}}}}' + # --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia.com/gpu","value":"","effect":"NoSchedule"}]}}}}' # should use the default profile # oc -n openshift-machine-api \ @@ -411,13 +407,13 @@ ocp_ack_upgrade_4.13(){ } ocp_gpu_taint_nodes(){ - oc adm taint node -l node-role.kubernetes.io/gpu nvidia-gpu-only=:NoSchedule --overwrite + oc adm taint node -l node-role.kubernetes.io/gpu nvidia.com/gpu=:NoSchedule --overwrite oc adm drain -l node-role.kubernetes.io/gpu --ignore-daemonsets --delete-emptydir-data oc adm uncordon -l node-role.kubernetes.io/gpu } ocp_gpu_untaint_nodes(){ - oc adm taint node -l node-role.kubernetes.io/gpu nvidia-gpu-only=:NoSchedule- + oc adm taint node -l node-role.kubernetes.io/gpu nvidia.com/gpu=:NoSchedule- } ocp_gpu_label_nodes_from_nfd(){ diff --git a/workshop/wip/parasol-insurance/01-setup/kustomization.yaml b/workshop/wip/parasol-insurance/01-setup/kustomization.yaml index 67fd9467..b4863ffa 100644 --- a/workshop/wip/parasol-insurance/01-setup/kustomization.yaml +++ b/workshop/wip/parasol-insurance/01-setup/kustomization.yaml @@ -33,9 +33,6 @@ patches: - effect: NoSchedule key: nvidia.com/gpu operator: Exists - - effect: NoSchedule - key: nvidia-gpu-only - operator: Exists - path: exclude-sc-ceph-rbd.yaml - path: exclude-sc-gp3.yaml - path: exclude-cm.yaml From 22e90ffa1acec475cbdccb9f16a69c9f6ecaa463 Mon Sep 17 00:00:00 2001 From: Cory Latschkowski Date: Thu, 3 Oct 2024 18:00:32 -0500 Subject: [PATCH 2/4] update: carve out gpu taint --- .../instance/components/aws-gpu-machineset/job.sh | 1 + .../instance/components/aws-gpu-machineset/ocp.sh | 13 +++++++++++++ scripts/library/ocp.sh | 12 ++++++++++++ 3 files changed, 26 insertions(+) diff --git a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/job.sh b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/job.sh index bf3d5bff..e47f0c22 100755 --- a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/job.sh +++ b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/job.sh @@ -8,3 +8,4 @@ INSTANCE_TYPE=${INSTANCE_TYPE:-g4dn.4xlarge} ocp_aws_cluster || exit 0 ocp_aws_create_gpu_machineset "${INSTANCE_TYPE}" ocp_create_machineset_autoscale +# ocp_aws_taint_gpu_machineset diff --git a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh index 7209e811..889217a6 100644 --- a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh +++ b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh @@ -6,6 +6,7 @@ # ocp_aws_cluster # ocp_aws_create_gpu_machineset # ocp_aws_clone_worker_machineset +# ocp_aws_taint_gpu_machineset # ocp_create_machineset_autoscale' # for function in $FUNCTIONS @@ -108,6 +109,18 @@ ocp_aws_clone_worker_machineset(){ --type=merge --patch '{"spec":{"template":{"spec":{"metadata":{"labels":{"node-role.kubernetes.io/'"${SHORT_NAME}"'":""}}}}}}' } +ocp_aws_taint_gpu_machineset(){ + INSTANCE_TYPE=${1:-g4dn.4xlarge} + MACHINE_SET_TYPE=$(oc -n openshift-machine-api get machinesets.machine.openshift.io -o name | grep "${INSTANCE_TYPE%.*}" | head -n1) + + echo "Patching: ${MACHINE_SET_TYPE}" + + # taint nodes for gpu-only workloads + oc -n openshift-machine-api \ + patch "${MACHINE_SET_TYPE}" \ + --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia.com/gpu","value":"","effect":"NoSchedule"}]}}}}' +} + ocp_create_machineset_autoscale(){ MACHINE_MIN=${1:-0} MACHINE_MAX=${2:-4} diff --git a/scripts/library/ocp.sh b/scripts/library/ocp.sh index 06e84975..f11b5505 100644 --- a/scripts/library/ocp.sh +++ b/scripts/library/ocp.sh @@ -268,6 +268,18 @@ ocp_aws_create_gpu_machineset(){ --type=merge --patch '{"spec":{"template":{"spec":{"providerSpec":{"value":{"instanceType":"'"${INSTANCE_TYPE}"'"}}}}}}' } +ocp_aws_taint_gpu_machineset(){ + INSTANCE_TYPE=${1:-g4dn.4xlarge} + MACHINE_SET_TYPE=$(oc -n openshift-machine-api get machinesets.machine.openshift.io -o name | grep "${INSTANCE_TYPE%.*}" | head -n1) + + echo "Patching: ${MACHINE_SET_TYPE}" + + # taint nodes for gpu-only workloads + oc -n openshift-machine-api \ + patch "${MACHINE_SET_TYPE}" \ + --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia.com/gpu","value":"","effect":"NoSchedule"}]}}}}' +} + ocp_create_machineset_autoscale(){ MACHINE_MIN=${1:-0} MACHINE_MAX=${2:-4} From a06ee594172055a95fa908eaeeb89dcd3553fad4 Mon Sep 17 00:00:00 2001 From: Cory Latschkowski Date: Thu, 3 Oct 2024 18:04:54 -0500 Subject: [PATCH 3/4] update: scripts --- .../instance/components/aws-gpu-machineset/ocp.sh | 5 +++-- scripts/library/ocp.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh index 889217a6..c13cc749 100644 --- a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh +++ b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh @@ -7,9 +7,10 @@ # ocp_aws_create_gpu_machineset # ocp_aws_clone_worker_machineset # ocp_aws_taint_gpu_machineset -# ocp_create_machineset_autoscale' +# ocp_create_machineset_autoscale +# ' -# for function in $FUNCTIONS +# for function in ${FUNCTIONS} # do # extract_function $function scripts/library/ocp.sh >> tmp # echo >> tmp diff --git a/scripts/library/ocp.sh b/scripts/library/ocp.sh index f11b5505..96ff3100 100644 --- a/scripts/library/ocp.sh +++ b/scripts/library/ocp.sh @@ -340,8 +340,8 @@ ocp_set_scheduler_profile(){ SCHED_PROFILE=${1:-LowNodeUtilization} # LowNodeUtilization, HighNodeUtilization, NoScoring - echo "see https://docs.openshift.com/container-platform/4.11/nodes/scheduling/nodes-scheduler-profiles.html" - echo "OPTIONS: LowNodeUtilization, HighNodeUtilization, NoScoring" + echo "see https://docs.openshift.com/container-platform/4.16/nodes/scheduling/nodes-scheduler-profiles.html" + echo "OPTIONS: LowNodeUtilization (default), HighNodeUtilization, NoScoring" echo "SCHED_PROFILE: ${SCHED_PROFILE}" oc patch schedulers.config.openshift.io/cluster --type merge --patch '{"spec":{"profile": "'"${SCHED_PROFILE}"'"}}' From 82272a991c3595a6918cb11b2f7e86568e9e5ebb Mon Sep 17 00:00:00 2001 From: Cory Latschkowski Date: Thu, 3 Oct 2024 18:14:45 -0500 Subject: [PATCH 4/4] cleanup --- .../instance/components/aws-gpu-machineset/ocp.sh | 5 ----- scripts/library/ocp.sh | 5 ----- 2 files changed, 10 deletions(-) diff --git a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh index c13cc749..54827216 100644 --- a/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh +++ b/components/operators/gpu-operator-certified/instance/components/aws-gpu-machineset/ocp.sh @@ -47,11 +47,6 @@ ocp_aws_create_gpu_machineset(){ oc -n openshift-machine-api \ patch "${MACHINE_SET_TYPE}" \ --type=merge --patch '{"spec":{"template":{"spec":{"metadata":{"labels":{"node-role.kubernetes.io/gpu":""}}}}}}' - - # # taint nodes for gpu-only workloads - # oc -n openshift-machine-api \ - # patch "${MACHINE_SET_TYPE}" \ - # --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia.com/gpu","value":"","effect":"NoSchedule"}]}}}}' # should use the default profile # oc -n openshift-machine-api \ diff --git a/scripts/library/ocp.sh b/scripts/library/ocp.sh index 96ff3100..420172a0 100644 --- a/scripts/library/ocp.sh +++ b/scripts/library/ocp.sh @@ -243,11 +243,6 @@ ocp_aws_create_gpu_machineset(){ oc -n openshift-machine-api \ patch "${MACHINE_SET_TYPE}" \ --type=merge --patch '{"spec":{"template":{"spec":{"metadata":{"labels":{"node-role.kubernetes.io/gpu":""}}}}}}' - - # # taint nodes for gpu-only workloads - # oc -n openshift-machine-api \ - # patch "${MACHINE_SET_TYPE}" \ - # --type=merge --patch '{"spec":{"template":{"spec":{"taints":[{"key":"nvidia.com/gpu","value":"","effect":"NoSchedule"}]}}}}' # should use the default profile # oc -n openshift-machine-api \