use kustomize to enable keystone webhook after kubeadm init

it does not add the --authorization-* arguments until after kubeadm init run. Once kubeadm init has finished, run kustomize to add the arguments to the kube-apiserver.yaml manifest. ref: kubernetes/cloud-provider-openstack#2575
0x00ace · Aug 2, 2024 · 7b87640 · 7b87640
1 parent b8dab75
commit 7b87640
Show file tree

Hide file tree

Showing 6 changed files with 95 additions and 89 deletions.
diff --git a/magnum_cluster_api/cmd/image_builder.py b/magnum_cluster_api/cmd/image_builder.py
@@ -67,7 +67,7 @@ def validate_version(_, __, value):
 @click.option(
     "--image-builder-version",
     show_default=True,
-    default="v0.1.30",
+    default="v0.1.31",
     help="Image builder tag (or commit) to use for building image",
 )
 @click.option(
@@ -165,67 +165,30 @@ def main(
         "node_custom_roles_pre": f"{node_custom_roles_pre}",
     }
 
-    # NOTE(mnaser): We use the latest tested daily ISO for Ubuntu 22.04 in order
-    #               to avoid a lengthy upgrade process.
-    if operating_system == "ubuntu-2204":
-        iso = "jammy-live-server-amd64.iso"
-
-        customization["iso_url"] = (
-            f"http://cdimage.ubuntu.com/ubuntu-server/jammy/daily-live/current/{iso}"
-        )
-
-        # Get the SHA256 sum for the ISO
-        r = requests.get(
-            "http://cdimage.ubuntu.com/ubuntu-server/jammy/daily-live/current/SHA256SUMS"
-        )
-        r.raise_for_status()
-        for line in r.text.splitlines():
-            if iso in line:
-                customization["iso_checksum"] = line.split()[0]
-                break
-
-        # Assert that we have the checksum
-        assert "iso_checksum" in customization
-
-    if operating_system == "rockylinux-8":
-        iso = "Rocky-x86_64-minimal.iso"
-
-        customization["iso_url"] = (
-            f"https://download.rockylinux.org/pub/rocky/8/isos/x86_64/{iso}"
-        )
-
-        # Get the SHA256 sum for the ISO
-        r = requests.get(
-            "https://download.rockylinux.org/pub/rocky/8/isos/x86_64/Rocky-x86_64-minimal.iso.CHECKSUM"
-        )
-        r.raise_for_status()
-        for line in r.text.splitlines():
-            if iso in line and "SHA256" in line:
-                customization["iso_checksum"] = line.split()[3]
-                break
-
-        # Assert that we have the checksum
-        assert "iso_checksum" in customization
-
-    if operating_system == "rockylinux-9":
-        iso = "Rocky-x86_64-minimal.iso"
-
-        customization["iso_url"] = (
-            f"https://download.rockylinux.org/pub/rocky/9/isos/x86_64/{iso}"
-        )
-
-        # Get the SHA256 sum for the ISO
-        r = requests.get(
-            "https://download.rockylinux.org/pub/rocky/9/isos/x86_64/Rocky-x86_64-minimal.iso.CHECKSUM"
-        )
-        r.raise_for_status()
-        for line in r.text.splitlines():
-            if iso in line and "SHA256" in line:
-                customization["iso_checksum"] = line.split()[3]
-                break
-
-        # Assert that we have the checksum
-        assert "iso_checksum" in customization
+    # NOTE(mnaser): Inside our CI, we use a local image in order speed up the
+    #               process, so we will not download the image from the internet.
+    if os.environ.get("CI") == "true":
+        if operating_system == "ubuntu-2204":
+            customization["iso_checksum"] = (
+                "https://static.atmosphere.dev/ubuntu/jammy/20240605.1/SHA256SUMS"
+            )
+            customization["iso_url"] = (
+                "https://static.atmosphere.dev/ubuntu/jammy/20240605.1/jammy-server-cloudimg-amd64.img"
+            )
+        elif operating_system == "rockylinux-8":
+            customization["iso_checksum"] = (
+                "https://static.atmosphere.dev/rocky/8/images/x86_64/CHECKSUM"
+            )
+            customization["iso_url"] = (
+                "https://static.atmosphere.dev/rocky/8/images/x86_64/Rocky-8-GenericCloud-Base.latest.x86_64.qcow2"
+            )
+        elif operating_system == "rockylinux-9":
+            customization["iso_checksum"] = (
+                "https://static.atmosphere.dev/rocky/9/images/x86_64/CHECKSUM"
+            )
+            customization["iso_url"] = (
+                "https://static.atmosphere.dev/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2"
+            )
 
     # NOTE(mnaser): Let's set number of CPUs to equal the number of CPUs on the
     #               host to speed up the build process.

diff --git a/magnum_cluster_api/conf.py b/magnum_cluster_api/conf.py
@@ -69,6 +69,16 @@
         default="$image_repository/cluster-autoscaler:v1.28.5",
         help="Image for the cluster auto-scaler for Kubernetes v1.28.",
     ),
+    cfg.StrOpt(
+        "v1_29_image",
+        default="$image_repository/cluster-autoscaler:v1.29.3",
+        help="Image for the cluster auto-scaler for Kubernetes v1.29.",
+    ),
+    cfg.StrOpt(
+        "v1_30_image",
+        default="$image_repository/cluster-autoscaler:v1.30.1",
+        help="Image for the cluster auto-scaler for Kubernetes v1.30.",
+    ),
 ]
 
 

diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py
@@ -736,6 +736,9 @@ def get_object(self) -> objects.KubeadmControlPlaneTemplate:
                                     "rm /var/lib/etcd/lost+found -rf",
                                     "bash /run/kubeadm/configure-kube-proxy.sh",
                                 ],
+                                "postKubeadmCommands": [
+                                    "echo PLACEHOLDER",
+                                ],
                             },
                         },
                     },
@@ -1999,18 +2002,46 @@ def get_object(self) -> objects.ClusterClass:
                                     "jsonPatches": [
                                         {
                                             "op": "add",
-                                            "path": "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/authentication-token-webhook-config-file",  # noqa: E501
-                                            "value": "/etc/kubernetes/webhooks/webhookconfig.yaml",
+                                            "path": "/spec/template/spec/kubeadmConfigSpec/files/-",
+                                            "value": {
+                                                "path": "/etc/kubernetes/keystone-kustomization/kustomization.yml",
+                                                "permissions": "0644",
+                                                "owner": "root:root",
+                                                "content": textwrap.dedent(
+                                                    """\
+                                                    resources:
+                                                    - kube-apiserver.yaml
+                                                    patches:
+                                                    - patch: |-
+                                                      - op: add
+                                                        path: /spec/containers/0/command/-
+                                                        value: --authentication-token-webhook-config-file=/etc/kubernetes/webhooks/webhookconfig.yaml
+                                                      - op: add
+                                                        path: /spec/containers/0/command/-
+                                                        value: --authorization-webhook-config-file=/etc/kubernetes/webhooks/webhookconfig.yaml
+                                                      - op: add
+                                                        path: /spec/containers/0/command/-
+                                                        value: --authorization-mode=Node,RBAC,Webhook
+                                                      target:
+                                                        kind: Pod
+                                                    """
+                                                ),
+                                            },
+                                        },
+                                        {
+                                            "op": "add",
+                                            "path": "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-",
+                                            "value": "mkdir /etc/kubernetes/keystone-kustomization",
                                         },
                                         {
                                             "op": "add",
-                                            "path": "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/authorization-webhook-config-file",  # noqa: E501
-                                            "value": "/etc/kubernetes/webhooks/webhookconfig.yaml",
+                                            "path": "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/-",
+                                            "value": "cp /etc/kubernetes/manifests/kube-apiserver.yaml /etc/kubernetes/keystone-kustomization/kube-apiserver.yaml",
                                         },
                                         {
                                             "op": "add",
-                                            "path": "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/authorization-mode",  # noqa: E501
-                                            "value": "Node,RBAC,Webhook",
+                                            "path": "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/-",
+                                            "value": "kubectl kustomize /etc/kubernetes/keystone-kustomization -o /etc/kubernetes/manifests/kube-apiserver.yaml",
                                         },
                                     ],
                                 }

diff --git a/zuul.d/jobs-flatcar.yaml b/zuul.d/jobs-flatcar.yaml
@@ -60,21 +60,21 @@
       kube_tag: v1.29.5
 
 - job:
-    name: magnum-cluster-api-sonobuoy-flatcar-v1.27.8-calico
-    parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.8
+    name: magnum-cluster-api-sonobuoy-flatcar-v1.27.14-calico
+    parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.14
     vars:
       network_driver: calico
 
 - job:
-    name: magnum-cluster-api-sonobuoy-flatcar-v1.27.8-cilium
-    parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.8
+    name: magnum-cluster-api-sonobuoy-flatcar-v1.27.14-cilium
+    parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.14
     vars:
       network_driver: cilium
 
 - project-template:
     name: magnum-cluster-api-flatcar
     check:
       jobs:
-        - magnum-cluster-api-image-build-flatcar-v1.27.8
-        - magnum-cluster-api-sonobuoy-flatcar-v1.27.8-calico
-        - magnum-cluster-api-sonobuoy-flatcar-v1.27.8-cilium
+        - magnum-cluster-api-image-build-flatcar-v1.27.14
+        - magnum-cluster-api-sonobuoy-flatcar-v1.27.14-calico
+        - magnum-cluster-api-sonobuoy-flatcar-v1.27.14-cilium
diff --git a/zuul.d/jobs-rockylinux-8.yaml b/zuul.d/jobs-rockylinux-8.yaml
@@ -60,21 +60,22 @@
       kube_tag: v1.29.5
 
 - job:
-    name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-calico
-    parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8
+    name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.14-calico
+    parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.14
     vars:
       network_driver: calico
 
 - job:
-    name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-cilium
-    parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8
+    name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.14-cilium
+    parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.14
     vars:
       network_driver: cilium
 
 - project-template:
     name: magnum-cluster-api-rockylinux-8
     check:
       jobs:
-        - magnum-cluster-api-image-build-rockylinux-8-v1.27.8
-        - magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-calico
-        - magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-cilium
+        - magnum-cluster-api-image-build-rockylinux-8-v1.27.14
+        - magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.14-calico
+        - magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.14-cilium
+
diff --git a/zuul.d/jobs-rockylinux-9.yaml b/zuul.d/jobs-rockylinux-9.yaml
@@ -60,21 +60,22 @@
       kube_tag: v1.29.5
 
 - job:
-    name: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8-calico
-    parent: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8
+    name: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.14-calico
+    parent: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.14
     vars:
       network_driver: calico
 
 - job:
-    name: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8-cilium
-    parent: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8
+    name: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.14-cilium
+    parent: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.14
     vars:
       network_driver: cilium
 
 - project-template:
     name: magnum-cluster-api-rockylinux-9
     check:
       jobs:
-        - magnum-cluster-api-image-build-rockylinux-9-v1.27.8
-        - magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8-calico
-        - magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8-cilium
+        - magnum-cluster-api-image-build-rockylinux-9-v1.27.14
+        - magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.14-calico
+        - magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.14-cilium
+