From f9bef8d607576aa33d4fca094051b84f6e85d67e Mon Sep 17 00:00:00 2001 From: Sergey Vasilyev Date: Sat, 9 May 2020 01:35:13 +0200 Subject: [PATCH 1/4] Document preservation of unknown fields in custom resources's statuses --- docs/configuration.rst | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 21703e86..2f4bdd67 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -252,11 +252,44 @@ For this, inherit from `kopf.StateStorage`, and implement its abstract methods The legacy behavior is an equivalent of ``kopf.StatusStateStorage(field='status.kopf.progress')``. - However, the ``.status`` stanza is not always stored by the server - for built-in or improperly configured custom resources since Kubernetes 1.16 - (see `#321 `_). - The new default "smart" engine is supposed to ensure a smooth upgrade + Starting with Kubernetes 1.16, both custom and built-in resources have + strict structural schemas with pruning of unknown fields + (more information is in `Future of CRDs: Structural Schemas`__). + + __ https://kubernetes.io/blog/2019/06/20/crd-structural-schema/ + + Long story short, unknown fields are silently pruned by Kubernetes API. + As a result, Kopf's status storage will not be able to actually store + anything in the resource, as it will be instantly lost. + (See `#321 `_.) + + To quickly fix this for custom resources, modify their definitions + with ``x-kubernetes-preserve-unknown-fields: true``. For example: + + .. code-block:: yaml + + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + spec: + scope: ... + group: ... + names: ... + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + + See a more verbose example in ``examples/crd.yaml``. + + For built-in resources, such as pods, namespaces, etc, the schemas cannot + be modified, so a full switch to annotations storage is advised. + + The new default "smart" storage is supposed to ensure a smooth upgrade of Kopf-based operators to the new state location without special upgrade actions or conversions needed. From bca9535cffada75540275d4d323974709480501e Mon Sep 17 00:00:00 2001 From: Sergey Vasilyev Date: Sat, 9 May 2020 01:35:57 +0200 Subject: [PATCH 2/4] Upgrade CRDs from v1beta1 to v1 for K8s 1.16+; keep remarks for K8s 1.15 --- docs/walkthrough/resources.rst | 41 +++++++++++++++++++++++++++ examples/crd-v1beta1.yaml | 39 ++++++++++++++++++++++++++ examples/crd.yaml | 51 ++++++++++++++++++++-------------- peering.yaml | 36 +++++++++++++++++------- 4 files changed, 136 insertions(+), 31 deletions(-) create mode 100644 examples/crd-v1beta1.yaml diff --git a/docs/walkthrough/resources.rst b/docs/walkthrough/resources.rst index 5d4aa74a..8ad64a66 100644 --- a/docs/walkthrough/resources.rst +++ b/docs/walkthrough/resources.rst @@ -7,6 +7,8 @@ Custom Resource Definition Let us define a CRD (custom resource definition) for our object. +For Kubernetes 1.15 and below: + .. code-block:: yaml :caption: crd.yaml :name: crd-yaml @@ -18,10 +20,38 @@ Let us define a CRD (custom resource definition) for our object. spec: scope: Namespaced group: zalando.org + names: + kind: EphemeralVolumeClaim + plural: ephemeralvolumeclaims + singular: ephemeralvolumeclaim + shortNames: + - evcs + - evc versions: - name: v1 served: true storage: true + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true + +For Kubernetes 1.16 and above: + +.. code-block:: yaml + :caption: crd.yaml + :name: crd-yaml + + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ephemeralvolumeclaims.zalando.org + spec: + scope: Namespaced + group: zalando.org names: kind: EphemeralVolumeClaim plural: ephemeralvolumeclaims @@ -29,6 +59,17 @@ Let us define a CRD (custom resource definition) for our object. shortNames: - evcs - evc + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true Note the short names: they can be used as the aliases on the command line, when getting a list or an object of that kind. diff --git a/examples/crd-v1beta1.yaml b/examples/crd-v1beta1.yaml new file mode 100644 index 00000000..93248ab4 --- /dev/null +++ b/examples/crd-v1beta1.yaml @@ -0,0 +1,39 @@ +# A demo CRD for the Kopf example operators. +# Use it with Kubernetes 1.15 and below. +# For Kubernetes 1.16 and above, use crd.yaml. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: kopfexamples.zalando.org +spec: + scope: Namespaced + group: zalando.org + versions: + - name: v1 + served: true + storage: true + names: + kind: KopfExample + plural: kopfexamples + singular: kopfexample + shortNames: + - kopfexes + - kopfex + - kexes + - kex + additionalPrinterColumns: + - name: Duration + type: string + priority: 0 + JSONPath: .spec.duration + description: For how long the pod should sleep. + - name: Children + type: string + priority: 0 + JSONPath: .status.create_fn.children + description: The children pods created. + - name: Message + type: string + priority: 0 + JSONPath: .status.create_fn.message + description: As returned from the handler (sometimes). diff --git a/examples/crd.yaml b/examples/crd.yaml index 8e00e193..eba6ea1e 100644 --- a/examples/crd.yaml +++ b/examples/crd.yaml @@ -1,15 +1,13 @@ # A demo CRD for the Kopf example operators. -apiVersion: apiextensions.k8s.io/v1beta1 +# Use it with Kubernetes 1.16 and above. +# For Kubernetes 1.15 and below, use crd-v1beta1.yaml. +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: kopfexamples.zalando.org spec: scope: Namespaced group: zalando.org - versions: - - name: v1 - served: true - storage: true names: kind: KopfExample plural: kopfexamples @@ -19,19 +17,30 @@ spec: - kopfex - kexes - kex - additionalPrinterColumns: - - name: Duration - type: string - priority: 0 - JSONPath: .spec.duration - description: For how long the pod should sleep. - - name: Children - type: string - priority: 0 - JSONPath: .status.create_fn.children - description: The children pods created. - - name: Message - type: string - priority: 0 - JSONPath: .status.create_fn.message - description: As returned from the handler (sometimes). + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalPrinterColumns: + - name: Duration + type: string + priority: 0 + jsonPath: .spec.duration + description: For how long the pod should sleep. + - name: Children + type: string + priority: 0 + jsonPath: .status.create_fn.children + description: The children pods created. + - name: Message + type: string + priority: 0 + jsonPath: .status.create_fn.message + description: As returned from the handler (sometimes). diff --git a/peering.yaml b/peering.yaml index ffc35f6b..01bae790 100644 --- a/peering.yaml +++ b/peering.yaml @@ -1,35 +1,51 @@ +# This file is for Kubernetes >= 1.16. +# For Kubernetes <= 1.15, change "v1" to "v1beta1", and remove schemas. --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clusterkopfpeerings.zalando.org spec: scope: Cluster group: zalando.org - versions: - - name: v1 - served: true - storage: true names: kind: ClusterKopfPeering plural: clusterkopfpeerings singular: clusterkopfpeering + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: kopfpeerings.zalando.org spec: scope: Namespaced group: zalando.org - versions: - - name: v1 - served: true - storage: true names: kind: KopfPeering plural: kopfpeerings singular: kopfpeering + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true --- apiVersion: zalando.org/v1 kind: ClusterKopfPeering From e0f6bf0a358f01d43765ac2c8ae7b07bb0cb7d13 Mon Sep 17 00:00:00 2001 From: Sergey Vasilyev Date: Mon, 11 May 2020 12:05:50 +0200 Subject: [PATCH 3/4] Permit arbitrary fields in `spec` of demo CRDs (same as it was before) --- docs/walkthrough/resources.rst | 6 ++++++ examples/crd.yaml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/docs/walkthrough/resources.rst b/docs/walkthrough/resources.rst index 8ad64a66..06d58bb0 100644 --- a/docs/walkthrough/resources.rst +++ b/docs/walkthrough/resources.rst @@ -35,6 +35,9 @@ For Kubernetes 1.15 and below: openAPIV3Schema: type: object properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true status: type: object x-kubernetes-preserve-unknown-fields: true @@ -67,6 +70,9 @@ For Kubernetes 1.16 and above: openAPIV3Schema: type: object properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true status: type: object x-kubernetes-preserve-unknown-fields: true diff --git a/examples/crd.yaml b/examples/crd.yaml index eba6ea1e..8acbbf92 100644 --- a/examples/crd.yaml +++ b/examples/crd.yaml @@ -25,6 +25,9 @@ spec: openAPIV3Schema: type: object properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true status: type: object x-kubernetes-preserve-unknown-fields: true From 61bde046c942969d24d2cb6e1e78c0cb256d0165 Mon Sep 17 00:00:00 2001 From: Sergey Vasilyev Date: Mon, 11 May 2020 12:27:45 +0200 Subject: [PATCH 4/4] Use proper v1 or v1beta1 APIs for KopfExample/KopfPeering in e2e tests --- .travis.yml | 9 ++-- examples/09-testing/test_example_09.py | 10 +++- .../11-filtering-handlers/test_example_11.py | 10 +++- peering-v1beta1.yaml | 46 +++++++++++++++++++ peering.yaml | 2 +- tests/e2e/conftest.py | 25 ++++++++-- 6 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 peering-v1beta1.yaml diff --git a/.travis.yml b/.travis.yml index c8df2bb6..d758cc0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,10 +19,11 @@ env: - KUBERNETES_VERSION=latest CLIENT=yes # only one "yes" is enough - KUBERNETES_VERSION=latest CLIENT=no - KUBERNETES_VERSION=v1.16.0 CLIENT=no - - KUBERNETES_VERSION=v1.15.0 CLIENT=no - - KUBERNETES_VERSION=v1.14.0 CLIENT=no - - KUBERNETES_VERSION=v1.13.0 CLIENT=no - - KUBERNETES_VERSION=v1.12.0 CLIENT=no + - KUBERNETES_VERSION=v1.16.0 CLIENT=no CRDAPI=v1beta1 + - KUBERNETES_VERSION=v1.15.0 CLIENT=no CRDAPI=v1beta1 + - KUBERNETES_VERSION=v1.14.0 CLIENT=no CRDAPI=v1beta1 + - KUBERNETES_VERSION=v1.13.0 CLIENT=no CRDAPI=v1beta1 + - KUBERNETES_VERSION=v1.12.0 CLIENT=no CRDAPI=v1beta1 # - KUBERNETES_VERSION=v1.11.10 # Minikube fails on CRI preflight checks # - KUBERNETES_VERSION=v1.10.13 # CRDs require spec.version, which fails on 1.14 diff --git a/examples/09-testing/test_example_09.py b/examples/09-testing/test_example_09.py index 90071156..27758e2e 100644 --- a/examples/09-testing/test_example_09.py +++ b/examples/09-testing/test_example_09.py @@ -6,13 +6,19 @@ import kopf.testing -crd_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'crd.yaml')) obj_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'obj.yaml')) example_py = os.path.relpath(os.path.join(os.path.dirname(__file__), 'example.py')) +@pytest.fixture(scope='session') +def crd_yaml(): + crd_api = os.environ.get('CRDAPI', 'v1') + crd_file = 'crd.yaml' if crd_api == 'v1' else f'crd-{crd_api}.yaml' + return os.path.relpath(os.path.join(os.path.dirname(__file__), '..', crd_file)) + + @pytest.fixture(autouse=True) -def crd_exists(): +def crd_exists(crd_yaml): subprocess.run(f"kubectl apply -f {crd_yaml}", check=True, timeout=10, capture_output=True, shell=True) diff --git a/examples/11-filtering-handlers/test_example_11.py b/examples/11-filtering-handlers/test_example_11.py index 8e5edd0c..4b88a7f9 100644 --- a/examples/11-filtering-handlers/test_example_11.py +++ b/examples/11-filtering-handlers/test_example_11.py @@ -6,13 +6,19 @@ import kopf.testing -crd_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'crd.yaml')) obj_yaml = os.path.relpath(os.path.join(os.path.dirname(__file__), '..', 'obj.yaml')) example_py = os.path.relpath(os.path.join(os.path.dirname(__file__), 'example.py')) +@pytest.fixture(scope='session') +def crd_yaml(): + crd_api = os.environ.get('CRDAPI', 'v1') + crd_file = 'crd.yaml' if crd_api == 'v1' else f'crd-{crd_api}.yaml' + return os.path.relpath(os.path.join(os.path.dirname(__file__), '..', crd_file)) + + @pytest.fixture(autouse=True) -def crd_exists(): +def crd_exists(crd_yaml): subprocess.run(f"kubectl apply -f {crd_yaml}", check=True, timeout=10, capture_output=True, shell=True) diff --git a/peering-v1beta1.yaml b/peering-v1beta1.yaml new file mode 100644 index 00000000..5df7e891 --- /dev/null +++ b/peering-v1beta1.yaml @@ -0,0 +1,46 @@ +# This file is for Kubernetes <= 1.15. +# For Kubernetes >= 1.16, use peering.yaml. +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterkopfpeerings.zalando.org +spec: + scope: Cluster + group: zalando.org + names: + kind: ClusterKopfPeering + plural: clusterkopfpeerings + singular: clusterkopfpeering + versions: + - name: v1 + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: kopfpeerings.zalando.org +spec: + scope: Namespaced + group: zalando.org + names: + kind: KopfPeering + plural: kopfpeerings + singular: kopfpeering + versions: + - name: v1 + served: true + storage: true +--- +apiVersion: zalando.org/v1 +kind: ClusterKopfPeering +metadata: + name: default +--- +apiVersion: zalando.org/v1 +kind: KopfPeering +metadata: + namespace: default + name: default +--- diff --git a/peering.yaml b/peering.yaml index 01bae790..cad2f20d 100644 --- a/peering.yaml +++ b/peering.yaml @@ -1,5 +1,5 @@ # This file is for Kubernetes >= 1.16. -# For Kubernetes <= 1.15, change "v1" to "v1beta1", and remove schemas. +# For Kubernetes <= 1.15, use peering-v1beta1.yaml. --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 1d532f9d..3f4e64a4 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,5 +1,7 @@ import glob import os.path +import re + import pathlib import subprocess @@ -16,15 +18,30 @@ def exampledir(request): return pathlib.Path(request.param) +@pytest.fixture(scope='session') +def peering_yaml(): + crd_api = os.environ.get('CRDAPI', 'v1') + crd_file = 'peering.yaml' if crd_api == 'v1' else f'peering-{crd_api}.yaml' + return f'{crd_file}' + + +@pytest.fixture(scope='session') +def crd_yaml(): + crd_api = os.environ.get('CRDAPI', 'v1') + crd_file = 'crd.yaml' if crd_api == 'v1' else f'crd-{crd_api}.yaml' + return f'examples/{crd_file}' + + @pytest.fixture() -def with_crd(): - subprocess.run("kubectl apply -f examples/crd.yaml", +def with_crd(crd_yaml): + # Our best guess on which Kubernetes version we are running on. + subprocess.run(f"kubectl apply -f {crd_yaml}", shell=True, check=True, timeout=10, capture_output=True) @pytest.fixture() -def with_peering(): - subprocess.run("kubectl apply -f peering.yaml", +def with_peering(peering_yaml): + subprocess.run(f"kubectl apply -f {peering_yaml}", shell=True, check=True, timeout=10, capture_output=True)