diff --git a/Makefile b/Makefile index 8a22fba..a49d03d 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,9 @@ testv: @go test ./... -cover -v -args --logtostderr -v=2 .PHONY: integration-test -integration-test: +integration-test: + # Uncomment to list verbose output + # @go test ./... -cover --tags=integration -v -args --logtostderr -v=1 @go test ./... -cover --tags=integration .PHONY: e2e-test diff --git a/go.mod b/go.mod index 854dbca..6fc53f4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module mayadata.io/d-operators go 1.13 require ( + github.com/ghodss/yaml v1.0.0 github.com/go-resty/resty/v2 v2.2.0 github.com/google/go-cmp v0.4.0 github.com/pkg/errors v0.9.1 diff --git a/pkg/recipe/apply.go b/pkg/recipe/apply.go index 739a325..56c80ce 100644 --- a/pkg/recipe/apply.go +++ b/pkg/recipe/apply.go @@ -18,11 +18,8 @@ package recipe import ( "fmt" - "strings" "github.com/pkg/errors" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -54,404 +51,33 @@ func NewApplier(config ApplyableConfig) *Applyable { } } -func (a *Applyable) postCreateCRDV1(crd *v1.CustomResourceDefinition) error { - if len(crd.Spec.Versions) == 0 { - return errors.Errorf( - "Invalid CRD spec: Missing spec.versions", - ) - } - var versionToVerify = crd.Spec.Versions[0].Name - message := fmt.Sprintf( - "PostCreate CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+versionToVerify, - ) - // Is custom resource definition discovered & - // can its resource(s) be listed - err := a.Retry.Waitf( - func() (bool, error) { - got := a.GetAPIForAPIVersionAndResource( - crd.Spec.Group+"/"+versionToVerify, - crd.Spec.Names.Plural, - ) - if got == nil { - return a.IsFailFastOnDiscoveryError(), - errors.Errorf( - "Failed to discover: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+versionToVerify, - ) - } - // fetch dynamic client for the custom resource - // corresponding to this CRD - customResourceClient, err := a.GetClientForAPIVersionAndResource( - crd.Spec.Group+"/"+versionToVerify, - crd.Spec.Names.Plural, - ) - if err != nil { - return a.IsFailFastOnDiscoveryError(), err - } - _, err = customResourceClient.List(metav1.ListOptions{}) - if err != nil { - return false, err - } - return true, nil - }, - message, - ) - return err -} - -func (a *Applyable) postCreateCRD( - crd *v1beta1.CustomResourceDefinition, -) error { - message := fmt.Sprintf( - "PostCreate CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+crd.Spec.Version, - ) - // discover custom resource API - err := a.Retry.Waitf( - func() (bool, error) { - got := a.GetAPIForAPIVersionAndResource( - crd.Spec.Group+"/"+crd.Spec.Version, - crd.Spec.Names.Plural, - ) - if got == nil { - return a.IsFailFastOnDiscoveryError(), - errors.Errorf( - "Failed to discover: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+crd.Spec.Version, - ) - } - // fetch dynamic client for the custom resource - // corresponding to this CRD - customResourceClient, err := a.GetClientForAPIVersionAndResource( - crd.Spec.Group+"/"+crd.Spec.Version, - crd.Spec.Names.Plural, - ) - if err != nil { - return a.IsFailFastOnDiscoveryError(), err - } - _, err = customResourceClient.List(metav1.ListOptions{}) - if err != nil { - return false, err - } - return true, nil - }, - message, - ) - return err -} - -func (a *Applyable) createCRDV1() (*types.ApplyResult, error) { - var crd *v1.CustomResourceDefinition - err := UnstructToTyped(a.Apply.State, &crd) - if err != nil { - return nil, err - } - // use crd client to create crd - crd, err = a.crdClientV1. - CustomResourceDefinitions(). - Create(crd) - if err != nil { - return nil, err - } - // add to teardown functions - a.AddToTeardown(func() error { - _, err := a.crdClientV1. - CustomResourceDefinitions(). - Get( - crd.GetName(), - metav1.GetOptions{}, - ) - if err != nil && apierrors.IsNotFound(err) { - // nothing to do - return nil - } - return a.crdClientV1. - CustomResourceDefinitions(). - Delete( - crd.Name, - nil, - ) - }) - // run an additional step to wait till this CRD - // is discovered at apiserver - err = a.postCreateCRDV1(crd) - if err != nil { - return nil, err - } - return &types.ApplyResult{ - Phase: types.ApplyStatusPassed, - Message: fmt.Sprintf( - "Create CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - a.Apply.State.GetAPIVersion(), - ), - }, nil -} - -func (a *Applyable) createCRD() (*types.ApplyResult, error) { - var crd *v1beta1.CustomResourceDefinition - err := UnstructToTyped(a.Apply.State, &crd) - if err != nil { - return nil, err - } - // use crd client to create crd - crd, err = a.crdClient. - CustomResourceDefinitions(). - Create(crd) - if err != nil { - return nil, err - } - // add to teardown functions - a.AddToTeardown(func() error { - _, err := a.crdClient. - CustomResourceDefinitions(). - Get( - crd.GetName(), - metav1.GetOptions{}, - ) - if err != nil && apierrors.IsNotFound(err) { - // nothing to do - return nil +func (a *Applyable) applyCRD() (*types.ApplyResult, error) { + if IsCRDVersion(a.Apply.State.GetAPIVersion(), "v1") { + e, err := NewCRDV1Executor(ExecutableCRDV1Config{ + BaseRunner: a.BaseRunner, + IgnoreDiscovery: a.Apply.IgnoreDiscovery, + State: a.Apply.State, + }) + if err != nil { + return nil, err } - return a.crdClient. - CustomResourceDefinitions(). - Delete( - crd.Name, - nil, - ) - }) - if !a.Apply.IgnoreDiscovery { - // run an additional step to wait till this CRD - // is discovered at apiserver - err = a.postCreateCRD(crd) + return e.Apply() + } else if IsCRDVersion(a.Apply.State.GetAPIVersion(), "v1beta1") { + e, err := NewCRDV1Beta1Executor(ExecutableCRDV1Beta1Config{ + BaseRunner: a.BaseRunner, + IgnoreDiscovery: a.Apply.IgnoreDiscovery, + State: a.Apply.State, + }) if err != nil { return nil, err } - } - return &types.ApplyResult{ - Phase: types.ApplyStatusPassed, - Message: fmt.Sprintf( - "Create CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+crd.Spec.Version, - ), - }, nil -} - -func (a *Applyable) updateCRDV1() (*types.ApplyResult, error) { - var crd *v1.CustomResourceDefinition - // transform to typed CRD to make use of crd client - err := UnstructToTyped(a.Apply.State, &crd) - if err != nil { - return nil, err - } - // get the CRD observed at the cluster - target, err := a.crdClientV1. - CustomResourceDefinitions(). - Get( - a.Apply.State.GetName(), - metav1.GetOptions{}, - ) - if err != nil { - return nil, err - } - // tansform back to unstruct type to run 3-way merge - targetAsUnstruct, err := TypedToUnstruct(target) - if err != nil { - return nil, err - } - merged := &unstructured.Unstructured{} - // 3-way merge - merged.Object, err = dynamicapply.Merge( - targetAsUnstruct.UnstructuredContent(), // observed - a.Apply.State.UnstructuredContent(), // last applied - a.Apply.State.UnstructuredContent(), // desired - ) - if err != nil { - return nil, err - } - // transform again to typed CRD to execute update - err = UnstructToTyped(merged, crd) - if err != nil { - return nil, err - } - // update the final merged state of CRD - // - // NOTE: - // At this point we are performing a server side - // apply against the CRD - _, err = a.crdClientV1. - CustomResourceDefinitions(). - Update( - crd, - ) - if err != nil { - return nil, err - } - return &types.ApplyResult{ - Phase: types.ApplyStatusPassed, - Message: fmt.Sprintf( - "Update CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, + return e.Apply() + } else { + return nil, errors.Errorf( + "Unsupported CRD API version %q", a.Apply.State.GetAPIVersion(), - ), - }, nil -} - -func (a *Applyable) updateCRD() (*types.ApplyResult, error) { - var crd *v1beta1.CustomResourceDefinition - // transform to typed CRD to make use of crd client - err := UnstructToTyped(a.Apply.State, &crd) - if err != nil { - return nil, err - } - // get the CRD observed at the cluster - target, err := a.crdClient. - CustomResourceDefinitions(). - Get( - a.Apply.State.GetName(), - metav1.GetOptions{}, ) - if err != nil { - return nil, err - } - // tansform back to unstruct type to run 3-way merge - targetAsUnstruct, err := TypedToUnstruct(target) - if err != nil { - return nil, err - } - merged := &unstructured.Unstructured{} - // 3-way merge - merged.Object, err = dynamicapply.Merge( - targetAsUnstruct.UnstructuredContent(), // observed - a.Apply.State.UnstructuredContent(), // last applied - a.Apply.State.UnstructuredContent(), // desired - ) - if err != nil { - return nil, err - } - // transform again to typed CRD to execute update - err = UnstructToTyped(merged, crd) - if err != nil { - return nil, err - } - // update the final merged state of CRD - // - // NOTE: - // At this point we are performing a server side - // apply against the CRD - _, err = a.crdClient. - CustomResourceDefinitions(). - Update( - crd, - ) - if err != nil { - return nil, err - } - return &types.ApplyResult{ - Phase: types.ApplyStatusPassed, - Message: fmt.Sprintf( - "Update CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+crd.Spec.Version, - ), - }, nil -} - -func (a *Applyable) applyCRDV1() (*types.ApplyResult, error) { - var crd *v1.CustomResourceDefinition - err := UnstructToTyped(a.Apply.State, &crd) - if err != nil { - return nil, err - } - message := fmt.Sprintf( - "Apply CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - a.Apply.State.GetAPIVersion(), - ) - // use crd client to get crd - err = a.Retry.Waitf( - func() (bool, error) { - _, err = a.crdClientV1. - CustomResourceDefinitions(). - Get( - crd.GetName(), - metav1.GetOptions{}, - ) - if err != nil { - if apierrors.IsNotFound(err) { - // condition exits since this is valid - return true, err - } - return false, err - } - return true, nil - }, - message, - ) - if err != nil { - if apierrors.IsNotFound(err) { - // this is a **create** operation - return a.createCRDV1() - } - return nil, err - } - // this is an **update** operation - return a.updateCRDV1() -} - -func (a *Applyable) applyCRD() (*types.ApplyResult, error) { - ver := a.Apply.State.GetAPIVersion() - if strings.HasSuffix(ver, "/v1") { - return a.applyCRDV1() - } - - var crd *v1beta1.CustomResourceDefinition - err := UnstructToTyped(a.Apply.State, &crd) - if err != nil { - return nil, err } - - // following code belongs to v1beta1 version - message := fmt.Sprintf( - "Apply CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+crd.Spec.Version, - ) - // use crd client to get crd - err = a.Retry.Waitf( - func() (bool, error) { - _, err = a.crdClient. - CustomResourceDefinitions(). - Get( - crd.GetName(), - metav1.GetOptions{}, - ) - if err != nil { - if apierrors.IsNotFound(err) { - // condition exits since this is valid - return true, err - } - return false, err - } - return true, nil - }, - message, - ) - if err != nil { - if apierrors.IsNotFound(err) { - // this is a **create** operation - return a.createCRD() - } - return nil, err - } - // this is an **update** operation - return a.updateCRD() } func (a *Applyable) createResource() (*types.ApplyResult, error) { diff --git a/pkg/recipe/apply_int_test.go b/pkg/recipe/apply_int_test.go new file mode 100644 index 0000000..c4ef0fb --- /dev/null +++ b/pkg/recipe/apply_int_test.go @@ -0,0 +1,335 @@ +// +build integration + +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + types "mayadata.io/d-operators/types/recipe" +) + +func TestApplyCRDV1(t *testing.T) { + state := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "cpools.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "scope": "Namespaced", + "names": map[string]interface{}{ + "kind": "CPool", + "listKind": "CPoolList", + "plural": "cpools", + "singular": "cpool", + "shortNames": []interface{}{ + "cp", + }, + }, + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + "served": true, + "storage": true, + "subresources": map[string]interface{}{ + "status": map[string]interface{}{}, + }, + "schema": map[string]interface{}{ + "openAPIV3Schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "apiVersion": map[string]interface{}{ + "type": "string", + }, + "kind": map[string]interface{}{ + "type": "string", + }, + "metadata": map[string]interface{}{ + "type": "object", + }, + "spec": map[string]interface{}{ + "description": "Specification of the mayastor pool.", + "type": "object", + "required": []interface{}{ + "node", + "disks", + }, + "properties": map[string]interface{}{ + "node": map[string]interface{}{ + "description": "Name of the k8s node where the storage pool is located.", + "type": "string", + }, + "disks": map[string]interface{}{ + "description": "Disk devices (paths or URIs) that should be used for the pool.", + "type": "array", + "items": map[string]interface{}{ + "type": "string", + }, + }, + }, + }, + "status": map[string]interface{}{ + "description": "Status part updated by the pool controller.", + "type": "object", + "properties": map[string]interface{}{ + "state": map[string]interface{}{ + "description": "Pool state.", + "type": "string", + }, + "reason": map[string]interface{}{ + "description": "Reason for the pool state value if applicable.", + "type": "string", + }, + "disks": map[string]interface{}{ + "description": "Disk device URIs that are actually used for the pool.", + "type": "array", + "items": map[string]interface{}{ + "type": "string", + }, + }, + "capacity": map[string]interface{}{ + "description": "Capacity of the pool in bytes.", + "type": "integer", + "format": "int64", + "minimum": int64(0), + }, + "used": map[string]interface{}{ + "description": "How many bytes are used in the pool.", + "type": "integer", + "format": "int64", + "minimum": int64(0), + }, + }, + }, + }, + }, + }, + "additionalPrinterColumns": []interface{}{ + map[string]interface{}{ + "name": "Node", + "type": "string", + "description": "Node where the storage pool is located", + "jsonPath": ".spec.node", + }, + map[string]interface{}{ + "name": "State", + "type": "string", + "description": "State of the storage pool", + "jsonPath": ".status.state", + }, + map[string]interface{}{ + "name": "Age", + "type": "date", + "jsonPath": ".metadata.creationTimestamp", + }, + }, + }, + }, + }, + }, + } + + br, err := NewDefaultBaseRunnerWithTeardown("apply crd testing") + if err != nil { + t.Fatalf( + "Failed to create kubernetes base runner: %v", + err, + ) + } + a := NewApplier(ApplyableConfig{ + BaseRunner: *br, + Apply: &types.Apply{ + State: state, + }, + }) + + result, err := a.Run() + if err != nil { + t.Fatalf( + "Error while testing crd create via applier: %v: %s", + err, + result, + ) + } + if result.Phase != types.ApplyStatusPassed { + t.Fatalf("Test failed while creating CRD via applier: %s", result) + } + + // --------------- + // UPDATE i.e. 3-WAY MERGE + // --------------- + update := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "cpools.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "names": map[string]interface{}{ + "plural": "cpools", + "shortNames": []interface{}{ + "cp", + "cpl", // new addition + }, + }, + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + }, + }, + }, + }, + } + a = NewApplier(ApplyableConfig{ + BaseRunner: *br, + Apply: &types.Apply{ + State: update, + }, + }) + if err != nil { + t.Fatalf( + "Failed to construct crd applier: %v", + err, + ) + } + + result, err = a.Run() + if err != nil { + t.Fatalf( + "Error while testing update via applier: %v: %s", + err, + result, + ) + } + if result.Phase != types.ApplyStatusPassed { + t.Fatalf("Test failed while updating CRD via applier: %s", result) + } +} + +func TestApplyCRDV1Beta1(t *testing.T) { + state := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1beta1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "soms.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "scope": "Namespaced", + "names": map[string]interface{}{ + "kind": "Som", + "listKind": "SomList", + "plural": "soms", + "singular": "som", + "shortNames": []interface{}{ + "som", + }, + }, + "version": "v1alpha1", + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + "served": true, + "storage": true, + }, + }, + }, + }, + } + + br, err := NewDefaultBaseRunnerWithTeardown("apply crd testing") + if err != nil { + t.Fatalf( + "Failed to create kubernetes base runner: %v", + err, + ) + } + a := NewApplier(ApplyableConfig{ + BaseRunner: *br, + Apply: &types.Apply{ + State: state, + }, + }) + + result, err := a.Run() + if err != nil { + t.Fatalf( + "Error while testing crd create via applier: %v: %s", + err, + result, + ) + } + if result.Phase != types.ApplyStatusPassed { + t.Fatalf("Test failed while creating CRD via applier: %s", result) + } + + // --------------- + // UPDATE i.e. 3-WAY MERGE + // --------------- + update := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1beta1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "soms.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "names": map[string]interface{}{ + "plural": "soms", + "shortNames": []interface{}{ + "som", + "somsom", // new addition + }, + }, + "version": "v1alpha1", + }, + }, + } + a = NewApplier(ApplyableConfig{ + BaseRunner: *br, + Apply: &types.Apply{ + State: update, + }, + }) + if err != nil { + t.Fatalf( + "Failed to construct crd applier: %v", + err, + ) + } + + result, err = a.Run() + if err != nil { + t.Fatalf( + "Error while testing update via applier: %v: %s", + err, + result, + ) + } + if result.Phase != types.ApplyStatusPassed { + t.Fatalf("Test failed while updating CRD via applier: %s", result) + } +} diff --git a/pkg/recipe/crd.go b/pkg/recipe/crd.go new file mode 100644 index 0000000..d5d7837 --- /dev/null +++ b/pkg/recipe/crd.go @@ -0,0 +1,248 @@ +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "fmt" + "strings" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + types "mayadata.io/d-operators/types/recipe" + dynamicapply "openebs.io/metac/dynamic/apply" +) + +// ExecutableCRD helps to apply or create desired CRD state +// against the cluster +type ExecutableCRD struct { + BaseRunner + IgnoreDiscovery bool + State *unstructured.Unstructured + DesiredCRVersion string + + // These are custom resource specific & are not + // related to CRD + CRResource string + CRAPIVersion string +} + +// IsCRDVersion returns true if provided version is +// set as a suffix of APIVersion +func IsCRDVersion(apiVersion, version string) bool { + return strings.HasSuffix(apiVersion, "/"+version) +} + +func (e *ExecutableCRD) postCreate() error { + message := fmt.Sprintf( + "PostCreate CRD: Resource %s: APIVersion %s", + e.CRResource, + e.CRAPIVersion, + ) + // discover custom resource API + err := e.Retry.Waitf( + func() (bool, error) { + got := e.GetAPIForAPIVersionAndResource( + e.CRAPIVersion, + e.CRResource, + ) + if got == nil { + return e.IsFailFastOnDiscoveryError(), + errors.Errorf( + "Failed to discover CRD: Resource %s: APIVersion %s", + e.CRResource, + e.CRAPIVersion, + ) + } + // fetch dynamic client for the custom resource + // corresponding to this CRD + cli, err := e.GetClientForAPIVersionAndResource( + e.CRAPIVersion, + e.CRResource, + ) + if err != nil { + return e.IsFailFastOnDiscoveryError(), err + } + // A successful list implies CRD is registered & is discovered + // in this binary + _, err = cli.List(metav1.ListOptions{}) + if err != nil { + return false, err + } + return true, nil + }, + message, + ) + return err +} + +// Create creates the CRD in Kubernetes cluster +func (e *ExecutableCRD) Create() (*types.CreateResult, error) { + cli, err := e.dynamicClientset.GetClientForAPIVersionAndKind( + e.State.GetAPIVersion(), // CRD APIVersion & not CR APIVersion + e.State.GetKind(), // CRD Kind & not CR Kind + ) + if err != nil { + return nil, err + } + + _, err = cli.Create(e.State, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + + // add to teardown functions + e.AddToTeardown(func() error { + _, err := cli.Get( + e.State.GetName(), + metav1.GetOptions{}, + ) + if err != nil && apierrors.IsNotFound(err) { + // nothing to do + return nil + } + return cli.Delete( + e.State.GetName(), + nil, + ) + }) + if !e.IgnoreDiscovery { + // run an additional step to wait till this CRD + // is discovered at apiserver + err = e.postCreate() + if err != nil { + return nil, err + } + } + return &types.CreateResult{ + Phase: types.CreateStatusPassed, + Message: fmt.Sprintf( + "Create CRD: Resource %s: APIVersion %s", + e.CRResource, + e.CRAPIVersion, + ), + }, nil +} + +// Update performs a 3-way merge of the CRD in the Kubernetes cluster +func (e *ExecutableCRD) Update() (*types.ApplyResult, error) { + cli, err := e.dynamicClientset.GetClientForAPIVersionAndKind( + e.State.GetAPIVersion(), // CRD APIVersion & not CR APIVersion + e.State.GetKind(), // CRD Kind & not CR Kind + ) + if err != nil { + return nil, err + } + + // Get the CRD observed at the cluster + target, err := cli.Get( + e.State.GetName(), + metav1.GetOptions{}, + ) + if err != nil { + return nil, err + } + + merged := &unstructured.Unstructured{} + // 3-way merge + merged.Object, err = dynamicapply.Merge( + target.UnstructuredContent(), // observed + e.State.UnstructuredContent(), // last applied + e.State.UnstructuredContent(), // desired + ) + if err != nil { + return nil, err + } + + // Update the final merged state of CRD + // + // NOTE: + // This is server side apply + _, err = cli.Update( + merged, + metav1.UpdateOptions{}, + ) + if err != nil { + return nil, err + } + + return &types.ApplyResult{ + Phase: types.ApplyStatusPassed, + Message: fmt.Sprintf( + "Update CRD: Resource %s: APIVersion %s", + e.CRResource, + e.CRAPIVersion, + ), + }, nil +} + +// Apply either creates or performs a 3-way merge of the CRD in +// the Kubernetes cluster +func (e *ExecutableCRD) Apply() (*types.ApplyResult, error) { + // following code belongs to v1beta1 version + message := fmt.Sprintf( + "Apply CRD: Resource %s: APIVersion %s", + e.CRResource, + e.CRAPIVersion, + ) + // use crd client to get crd + err := e.Retry.Waitf( + func() (bool, error) { + cli, err := + e.dynamicClientset.GetClientForAPIVersionAndKind( + e.State.GetAPIVersion(), // CRD APIVersion & not CR APIVersion + e.State.GetKind(), // CRD Kind & not CR Kind + ) + if err != nil { + return false, err + } + // Get the CRD observed at the cluster + _, err = cli.Get( + e.State.GetName(), + metav1.GetOptions{}, + ) + if err != nil { + if apierrors.IsNotFound(err) { + // condition exits since this is valid + return true, err + } + return false, err + } + return true, nil + }, + message, + ) + if err != nil { + if apierrors.IsNotFound(err) { + // this is a **create** operation + got, err := e.Create() + if err != nil { + return nil, err + } + return &types.ApplyResult{ + Phase: got.Phase.ToApplyStatusPhase(), + Message: got.Message, + Verbose: got.Verbose, + Warning: got.Warning, + }, nil + } + return nil, err + } + // this is an **update** operation + return e.Update() +} diff --git a/pkg/recipe/crd_test.go b/pkg/recipe/crd_test.go new file mode 100644 index 0000000..ee1438f --- /dev/null +++ b/pkg/recipe/crd_test.go @@ -0,0 +1,124 @@ +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "testing" + + gy "github.com/ghodss/yaml" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +var crd = `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: mayastorpools.openebs.io +spec: + group: openebs.io + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + description: Specification of the mayastor pool. + type: object + required: + - node + - disks + properties: + node: + description: Name of the k8s node where the storage pool is located. + type: string + disks: + description: Disk devices (paths or URIs) that should be used for the pool. + type: array + items: + type: string + status: + description: Status part updated by the pool controller. + type: object + properties: + state: + description: Pool state. + type: string + reason: + description: Reason for the pool state value if applicable. + type: string + disks: + description: Disk device URIs that are actually used for the pool. + type: array + items: + type: string + capacity: + description: Capacity of the pool in bytes. + type: integer + format: int64 + minimum: 0 + used: + description: How many bytes are used in the pool. + type: integer + format: int64 + minimum: 0 + additionalPrinterColumns: + - name: Node + type: string + description: Node where the storage pool is located + jsonPath: .spec.node + - name: State + type: string + description: State of the storage pool + jsonPath: .status.state + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + scope: Namespaced + names: + kind: MayastorPool + listKind: MayastorPoolList + plural: mayastorpools + singular: mayastorpool + shortNames: ["msp"] +` + +func TestMSPCRDV1FromStringUnmarshal(t *testing.T) { + var unstructObj unstructured.Unstructured + err := gy.Unmarshal([]byte(crd), &unstructObj) + if err != nil { + t.Fatalf("Failed to unmarshal: %v", err) + } + + var crdObj v1.CustomResourceDefinition + err = UnstructToTyped(&unstructObj, &crdObj) + if err == nil { + t.Fatal( + "Convert to CRD type: Expected error got none", + ) + } +} diff --git a/pkg/recipe/crd_v1.go b/pkg/recipe/crd_v1.go new file mode 100644 index 0000000..55dcb21 --- /dev/null +++ b/pkg/recipe/crd_v1.go @@ -0,0 +1,122 @@ +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "fmt" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// ExecutableCRDV1 helps to apply or create desired CRD state +// against the cluster +type ExecutableCRDV1 struct { + ExecutableCRD +} + +// ExecutableCRDV1Config helps in creating new instance of +// ExecutableCRDV1 +type ExecutableCRDV1Config struct { + BaseRunner BaseRunner + IgnoreDiscovery bool + State *unstructured.Unstructured + DesiredCRVersion string +} + +// NewCRDV1Executor returns a new instance of ExecutableCRDV1Beta1 +func NewCRDV1Executor(config ExecutableCRDV1Config) (*ExecutableCRDV1, error) { + e := &ExecutableCRDV1{ + ExecutableCRD: ExecutableCRD{ + BaseRunner: config.BaseRunner, + IgnoreDiscovery: config.IgnoreDiscovery, + State: config.State, + DesiredCRVersion: config.DesiredCRVersion, + }, + } + err := e.setCRResourceAndAPIVersion() + if err != nil { + return nil, err + } + return e, nil +} + +func (e *ExecutableCRDV1) setCRResourceAndAPIVersion() error { + plural, found, err := unstructured.NestedString( + e.State.Object, + "spec", + "names", + "plural", + ) + if err != nil { + return errors.Wrapf(err, "Failed to get spec.names.plural") + } + if !found { + return errors.Errorf("Missing spec.names.plural") + } + group, found, err := unstructured.NestedString( + e.State.Object, + "spec", + "group", + ) + if err != nil { + return errors.Wrapf(err, "Failed to get spec.group") + } + if !found { + return errors.Errorf("Missing spec.group") + } + // Get the version that is found first in the list + // if nothing has been set + if e.DesiredCRVersion == "" { + vers, found, err := unstructured.NestedSlice( + e.State.Object, + "spec", + "versions", + ) + if err != nil { + return errors.Wrapf(err, "Failed to get spec.versions") + } + if !found { + return errors.Errorf("Missing spec.versions") + } + for _, item := range vers { + itemObj, ok := item.(map[string]interface{}) + if !ok { + return errors.Errorf( + "Expected spec.versions type as map[string]interface{} got %T", + item, + ) + } + e.DesiredCRVersion = itemObj["name"].(string) + // -- + // First version in the list is only considered + // This value is only used for CRD discovery which + // again is an optional feature + // -- + break + } + } + + apiver := fmt.Sprintf("%s/%s", group, e.DesiredCRVersion) + + // memoize + e.CRResource = plural + e.CRAPIVersion = apiver + + // return resource name & apiVersion of the resource + return nil +} diff --git a/pkg/recipe/crd_v1_int_test.go b/pkg/recipe/crd_v1_int_test.go new file mode 100644 index 0000000..65b6fb9 --- /dev/null +++ b/pkg/recipe/crd_v1_int_test.go @@ -0,0 +1,230 @@ +// +build integration + +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + types "mayadata.io/d-operators/types/recipe" +) + +func TestCRDApply(t *testing.T) { + state := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "mayastorpools.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "scope": "Namespaced", + "names": map[string]interface{}{ + "kind": "MayastorPool", + "listKind": "MayastorPoolList", + "plural": "mayastorpools", + "singular": "mayastorpool", + "shortNames": []interface{}{ + "msp", + }, + }, + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + "served": true, + "storage": true, + "subresources": map[string]interface{}{ + "status": map[string]interface{}{}, + }, + "schema": map[string]interface{}{ + "openAPIV3Schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "apiVersion": map[string]interface{}{ + "type": "string", + }, + "kind": map[string]interface{}{ + "type": "string", + }, + "metadata": map[string]interface{}{ + "type": "object", + }, + "spec": map[string]interface{}{ + "description": "Specification of the mayastor pool.", + "type": "object", + "required": []interface{}{ + "node", + "disks", + }, + "properties": map[string]interface{}{ + "node": map[string]interface{}{ + "description": "Name of the k8s node where the storage pool is located.", + "type": "string", + }, + "disks": map[string]interface{}{ + "description": "Disk devices (paths or URIs) that should be used for the pool.", + "type": "array", + "items": map[string]interface{}{ + "type": "string", + }, + }, + }, + }, + "status": map[string]interface{}{ + "description": "Status part updated by the pool controller.", + "type": "object", + "properties": map[string]interface{}{ + "state": map[string]interface{}{ + "description": "Pool state.", + "type": "string", + }, + "reason": map[string]interface{}{ + "description": "Reason for the pool state value if applicable.", + "type": "string", + }, + "disks": map[string]interface{}{ + "description": "Disk device URIs that are actually used for the pool.", + "type": "array", + "items": map[string]interface{}{ + "type": "string", + }, + }, + "capacity": map[string]interface{}{ + "description": "Capacity of the pool in bytes.", + "type": "integer", + "format": "int64", + "minimum": int64(0), + }, + "used": map[string]interface{}{ + "description": "How many bytes are used in the pool.", + "type": "integer", + "format": "int64", + "minimum": int64(0), + }, + }, + }, + }, + }, + }, + "additionalPrinterColumns": []interface{}{ + map[string]interface{}{ + "name": "Node", + "type": "string", + "description": "Node where the storage pool is located", + "jsonPath": ".spec.node", + }, + map[string]interface{}{ + "name": "State", + "type": "string", + "description": "State of the storage pool", + "jsonPath": ".status.state", + }, + map[string]interface{}{ + "name": "Age", + "type": "date", + "jsonPath": ".metadata.creationTimestamp", + }, + }, + }, + }, + }, + }, + } + + br, err := NewDefaultBaseRunnerWithTeardown("apply crd testing") + if err != nil { + t.Fatalf( + "Failed to create kubernetes base runner: %v", + err, + ) + } + e, err := NewCRDV1Executor(ExecutableCRDV1Config{ + BaseRunner: *br, + State: state, + }) + if err != nil { + t.Fatalf( + "Failed to construct crd executor: %v", + err, + ) + } + + result, err := e.Apply() + if err != nil { + t.Fatalf( + "Error while testing create via apply: %v: %s", + err, + result, + ) + } + if result.Phase != types.ApplyStatusPassed { + t.Fatalf("Test failed while creating CRD via apply: %s", result) + } + + // --------------- + // UPDATE i.e. 3-WAY MERGE + // --------------- + update := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "mayastorpools.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "names": map[string]interface{}{ + "plural": "mayastorpools", + "shortNames": []interface{}{ + "msp", + "mayasp", // new addition + }, + }, + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + }, + }, + }, + }, + } + e, err = NewCRDV1Executor(ExecutableCRDV1Config{ + BaseRunner: *br, + State: update, + }) + if err != nil { + t.Fatalf( + "Failed to construct crd executor: %v", + err, + ) + } + + result, err = e.Apply() + if err != nil { + t.Fatalf( + "Error while testing update via apply: %v: %s", + err, + result, + ) + } + if result.Phase != types.ApplyStatusPassed { + t.Fatalf("Test failed while updating CRD via apply: %s", result) + } +} diff --git a/pkg/recipe/crd_v1beta1.go b/pkg/recipe/crd_v1beta1.go new file mode 100644 index 0000000..aa4945f --- /dev/null +++ b/pkg/recipe/crd_v1beta1.go @@ -0,0 +1,99 @@ +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "fmt" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// ExecutableCRDV1Beta1 helps to apply or create desired CRD state +// against the cluster +type ExecutableCRDV1Beta1 struct { + ExecutableCRD +} + +// ExecutableCRDV1Beta1Config helps in creating new instance of +// ExecutableCRDV1Beta1 +type ExecutableCRDV1Beta1Config struct { + BaseRunner BaseRunner + IgnoreDiscovery bool + State *unstructured.Unstructured +} + +// NewCRDV1Beta1Executor returns a new instance of ExecutableCRDV1Beta1 +func NewCRDV1Beta1Executor(config ExecutableCRDV1Beta1Config) (*ExecutableCRDV1Beta1, error) { + e := &ExecutableCRDV1Beta1{ + ExecutableCRD: ExecutableCRD{ + BaseRunner: config.BaseRunner, + IgnoreDiscovery: config.IgnoreDiscovery, + State: config.State, + }, + } + err := e.setCRResourceAndAPIVersion() + if err != nil { + return nil, err + } + return e, nil +} + +func (e *ExecutableCRDV1Beta1) setCRResourceAndAPIVersion() error { + plural, found, err := unstructured.NestedString( + e.State.Object, + "spec", + "names", + "plural", + ) + if err != nil { + return errors.Wrapf(err, "Failed to get spec.names.plural") + } + if !found { + return errors.Errorf("Missing spec.names.plural") + } + group, found, err := unstructured.NestedString( + e.State.Object, + "spec", + "group", + ) + if err != nil { + return errors.Wrapf(err, "Failed to get spec.group") + } + if !found { + return errors.Errorf("Missing spec.group") + } + ver, found, err := unstructured.NestedString( + e.State.Object, + "spec", + "version", + ) + if err != nil { + return errors.Wrapf(err, "Failed to get spec.version") + } + if !found || ver == "" { + return errors.Errorf("Missing spec.version") + } + apiver := fmt.Sprintf("%s/%s", group, ver) + + // memoize + e.CRResource = plural + e.CRAPIVersion = apiver + + // return resource name & apiVersion of the resource + return nil +} diff --git a/pkg/recipe/crd_v1beta1_int_test.go b/pkg/recipe/crd_v1beta1_int_test.go new file mode 100644 index 0000000..87b9b1c --- /dev/null +++ b/pkg/recipe/crd_v1beta1_int_test.go @@ -0,0 +1,135 @@ +// +build integration + +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + types "mayadata.io/d-operators/types/recipe" +) + +func TestCRDV1Beta1Apply(t *testing.T) { + state := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1beta1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "somethings.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "scope": "Namespaced", + "names": map[string]interface{}{ + "kind": "SomeThing", + "listKind": "SomeThingList", + "plural": "somethings", + "singular": "something", + "shortNames": []interface{}{ + "sme", + }, + }, + "version": "v1alpha1", + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + "served": true, + "storage": true, + }, + }, + }, + }, + } + + br, err := NewDefaultBaseRunnerWithTeardown("apply crd testing") + if err != nil { + t.Fatalf( + "Failed to create kubernetes base runner: %v", + err, + ) + } + e, err := NewCRDV1Beta1Executor(ExecutableCRDV1Beta1Config{ + BaseRunner: *br, + State: state, + }) + if err != nil { + t.Fatalf( + "Failed to construct crd v1beta1 executor: %v", + err, + ) + } + + result, err := e.Apply() + if err != nil { + t.Fatalf( + "Error while testing create CRD via apply: %v: %s", + err, + result, + ) + } + if result.Phase != types.ApplyStatusPassed { + t.Fatalf("Test failed while creating CRD via apply: %s", result) + } + + // --------------- + // UPDATE i.e. 3-WAY MERGE + // --------------- + update := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1beta1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "somethings.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "names": map[string]interface{}{ + "plural": "somethings", + "shortNames": []interface{}{ + "sme", + "smethng", + }, + }, + "version": "v1alpha1", + }, + }, + } + e, err = NewCRDV1Beta1Executor(ExecutableCRDV1Beta1Config{ + BaseRunner: *br, + State: update, + }) + if err != nil { + t.Fatalf( + "Failed to construct crd v1beta1 executor: %v", + err, + ) + } + + result, err = e.Apply() + if err != nil { + t.Fatalf( + "Error while testing update CRD via apply: %v: %s", + err, + result, + ) + } + if result.Phase != types.ApplyStatusPassed { + t.Fatalf("Test failed while updating CRD via apply: %s", result) + } +} diff --git a/pkg/recipe/create.go b/pkg/recipe/create.go index 954dba6..f7b8438 100644 --- a/pkg/recipe/create.go +++ b/pkg/recipe/create.go @@ -18,11 +18,8 @@ package recipe import ( "fmt" - "strings" "github.com/pkg/errors" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -67,203 +64,33 @@ func NewCreator(config CreatableConfig) *Creatable { } } -func (c *Creatable) postCreateCRDV1( - crd *v1.CustomResourceDefinition, -) error { - if len(crd.Spec.Versions) == 0 { - return errors.Errorf( - "Invalid CRD spec: Missing spec.versions", - ) - } - var versionToVerify = crd.Spec.Versions[0].Name - message := fmt.Sprintf( - "PostCreate CRD: Kind %s: APIVersion %s: TaskName %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+versionToVerify, - c.TaskName, - ) - // discover custom resource API - return c.Retry.Waitf( - func() (bool, error) { - api := c.GetAPIForAPIVersionAndResource( - crd.Spec.Group+"/"+versionToVerify, - crd.Spec.Names.Plural, - ) - if api == nil { - return c.IsFailFastOnDiscoveryError(), - errors.Errorf( - "Failed to discover: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+versionToVerify, - ) - } - // fetch dynamic client for the custom resource - // corresponding to this CRD - customResourceClient, err := c.GetClientForAPIVersionAndResource( - crd.Spec.Group+"/"+versionToVerify, - crd.Spec.Names.Plural, - ) - if err != nil { - return c.IsFailFastOnDiscoveryError(), err - } - _, err = customResourceClient.List(metav1.ListOptions{}) - if err != nil { - return false, err - } - return true, nil - }, - message, - ) -} - -func (c *Creatable) postCreateCRD( - crd *v1beta1.CustomResourceDefinition, -) error { - message := fmt.Sprintf( - "PostCreate CRD: Kind %s: APIVersion %s: TaskName %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+crd.Spec.Version, - c.TaskName, - ) - // discover custom resource API - return c.Retry.Waitf( - func() (bool, error) { - api := c.GetAPIForAPIVersionAndResource( - crd.Spec.Group+"/"+crd.Spec.Version, - crd.Spec.Names.Plural, - ) - if api == nil { - return c.IsFailFastOnDiscoveryError(), - errors.Errorf( - "Failed to discover: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+crd.Spec.Version, - ) - } - // fetch dynamic client for the custom resource - // corresponding to this CRD - customResourceClient, err := c.GetClientForAPIVersionAndResource( - crd.Spec.Group+"/"+crd.Spec.Version, - crd.Spec.Names.Plural, - ) - if err != nil { - return c.IsFailFastOnDiscoveryError(), err - } - _, err = customResourceClient.List(metav1.ListOptions{}) - if err != nil { - return false, err - } - return true, nil - }, - message, - ) -} - -func (c *Creatable) createCRDV1() (*types.CreateResult, error) { - var crd *v1.CustomResourceDefinition - err := UnstructToTyped(c.Create.State, &crd) - if err != nil { - return nil, err - } - // use crd client to create crd - crd, err = c.crdClientV1. - CustomResourceDefinitions(). - Create(crd) - if err != nil { - return nil, errors.Wrapf( - err, - "%s", - c, - ) - } - // add to teardown functions - c.AddToTeardown(func() error { - _, err := c.crdClientV1. - CustomResourceDefinitions(). - Get( - crd.GetName(), - metav1.GetOptions{}, - ) - if err != nil && apierrors.IsNotFound(err) { - // nothing to do - return nil - } - return c.crdClientV1. - CustomResourceDefinitions(). - Delete( - crd.Name, - nil, - ) - }) - // run an additional step to wait till this CRD - // is discovered at apiserver - err = c.postCreateCRDV1(crd) - if err != nil { - return nil, err - } - return &types.CreateResult{ - Phase: types.CreateStatusPassed, - Message: fmt.Sprintf( - "Create CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - c.Create.State.GetAPIVersion(), - ), - }, nil -} - func (c *Creatable) createCRD() (*types.CreateResult, error) { - ver := c.Create.State.GetAPIVersion() - if strings.HasSuffix(ver, "/v1") { - return c.createCRDV1() - } - - var crd *v1beta1.CustomResourceDefinition - err := UnstructToTyped(c.Create.State, &crd) - if err != nil { - return nil, err - } - // use crd client to create crd - crd, err = c.crdClient. - CustomResourceDefinitions(). - Create(crd) - if err != nil { - return nil, errors.Wrapf(err, "%s", c) - } - // add to teardown functions - c.AddToTeardown(func() error { - _, err := c.crdClient. - CustomResourceDefinitions(). - Get( - crd.GetName(), - metav1.GetOptions{}, - ) - if err != nil && apierrors.IsNotFound(err) { - // nothing to do - return nil + if IsCRDVersion(c.Create.State.GetAPIVersion(), "v1") { + e, err := NewCRDV1Executor(ExecutableCRDV1Config{ + BaseRunner: c.BaseRunner, + IgnoreDiscovery: c.Create.IgnoreDiscovery, + State: c.Create.State, + }) + if err != nil { + return nil, err } - return c.crdClient. - CustomResourceDefinitions(). - Delete( - crd.Name, - nil, - ) - }) - if !c.Create.IgnoreDiscovery { - // run an additional step to wait till this CRD - // is discovered at apiserver - err = c.postCreateCRD(crd) + return e.Create() + } else if IsCRDVersion(c.Create.State.GetAPIVersion(), "v1beta1") { + e, err := NewCRDV1Beta1Executor(ExecutableCRDV1Beta1Config{ + BaseRunner: c.BaseRunner, + IgnoreDiscovery: c.Create.IgnoreDiscovery, + State: c.Create.State, + }) if err != nil { return nil, err } + return e.Create() + } else { + return nil, errors.Errorf( + "Unsupported CRD API version %q", + c.Create.State.GetAPIVersion(), + ) } - return &types.CreateResult{ - Phase: types.CreateStatusPassed, - Message: fmt.Sprintf( - "Create CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+crd.Spec.Version, - ), - }, nil } func (c *Creatable) createResource( diff --git a/pkg/recipe/create_int_test.go b/pkg/recipe/create_int_test.go new file mode 100644 index 0000000..6a4a05e --- /dev/null +++ b/pkg/recipe/create_int_test.go @@ -0,0 +1,235 @@ +// +build integration + +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + types "mayadata.io/d-operators/types/recipe" +) + +func TestCreateCRDV1(t *testing.T) { + state := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "dpools.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "scope": "Namespaced", + "names": map[string]interface{}{ + "kind": "DPool", + "listKind": "DPoolList", + "plural": "dpools", + "singular": "dpool", + "shortNames": []interface{}{ + "dp", + }, + }, + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + "served": true, + "storage": true, + "subresources": map[string]interface{}{ + "status": map[string]interface{}{}, + }, + "schema": map[string]interface{}{ + "openAPIV3Schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "apiVersion": map[string]interface{}{ + "type": "string", + }, + "kind": map[string]interface{}{ + "type": "string", + }, + "metadata": map[string]interface{}{ + "type": "object", + }, + "spec": map[string]interface{}{ + "description": "Specification of the mayastor pool.", + "type": "object", + "required": []interface{}{ + "node", + "disks", + }, + "properties": map[string]interface{}{ + "node": map[string]interface{}{ + "description": "Name of the k8s node where the storage pool is located.", + "type": "string", + }, + "disks": map[string]interface{}{ + "description": "Disk devices (paths or URIs) that should be used for the pool.", + "type": "array", + "items": map[string]interface{}{ + "type": "string", + }, + }, + }, + }, + "status": map[string]interface{}{ + "description": "Status part updated by the pool controller.", + "type": "object", + "properties": map[string]interface{}{ + "state": map[string]interface{}{ + "description": "Pool state.", + "type": "string", + }, + "reason": map[string]interface{}{ + "description": "Reason for the pool state value if applicable.", + "type": "string", + }, + "disks": map[string]interface{}{ + "description": "Disk device URIs that are actually used for the pool.", + "type": "array", + "items": map[string]interface{}{ + "type": "string", + }, + }, + "capacity": map[string]interface{}{ + "description": "Capacity of the pool in bytes.", + "type": "integer", + "format": "int64", + "minimum": int64(0), + }, + "used": map[string]interface{}{ + "description": "How many bytes are used in the pool.", + "type": "integer", + "format": "int64", + "minimum": int64(0), + }, + }, + }, + }, + }, + }, + "additionalPrinterColumns": []interface{}{ + map[string]interface{}{ + "name": "Node", + "type": "string", + "description": "Node where the storage pool is located", + "jsonPath": ".spec.node", + }, + map[string]interface{}{ + "name": "State", + "type": "string", + "description": "State of the storage pool", + "jsonPath": ".status.state", + }, + map[string]interface{}{ + "name": "Age", + "type": "date", + "jsonPath": ".metadata.creationTimestamp", + }, + }, + }, + }, + }, + }, + } + + br, err := NewDefaultBaseRunnerWithTeardown("apply crd testing") + if err != nil { + t.Fatalf( + "Failed to create kubernetes base runner: %v", + err, + ) + } + c := NewCreator(CreatableConfig{ + BaseRunner: *br, + Create: &types.Create{ + State: state, + }, + }) + + result, err := c.Run() + if err != nil { + t.Fatalf( + "Error while testing crd create via creator: %v: %s", + err, + result, + ) + } + if result.Phase != types.CreateStatusPassed { + t.Fatalf("Test failed while creating CRD via creator: %s", result) + } +} + +func TestCreateCRDV1Beta1(t *testing.T) { + state := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1beta1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "doms.openebs.io", + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "scope": "Namespaced", + "names": map[string]interface{}{ + "kind": "Dom", + "listKind": "DomList", + "plural": "doms", + "singular": "dom", + "shortNames": []interface{}{ + "dom", + }, + }, + "version": "v1alpha1", + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + "served": true, + "storage": true, + }, + }, + }, + }, + } + + br, err := NewDefaultBaseRunnerWithTeardown("apply crd testing") + if err != nil { + t.Fatalf( + "Failed to create kubernetes base runner: %v", + err, + ) + } + c := NewCreator(CreatableConfig{ + BaseRunner: *br, + Create: &types.Create{ + State: state, + }, + }) + + result, err := c.Run() + if err != nil { + t.Fatalf( + "Error while testing crd create via creator: %v: %s", + err, + result, + ) + } + if result.Phase != types.CreateStatusPassed { + t.Fatalf("Test failed while creating CRD via creator: %s", result) + } +} diff --git a/pkg/recipe/fixture.go b/pkg/recipe/fixture.go index 6c6678e..7883135 100644 --- a/pkg/recipe/fixture.go +++ b/pkg/recipe/fixture.go @@ -218,8 +218,11 @@ func (f *Fixture) GetClientForAPIVersionAndResource( ) } -// GetAPIForAPIVersionAndResource returns the discovered api based -// on the provided api version & resource +// GetAPIForAPIVersionAndResource returns the discovered +// api based on the provided api version & resource +// +// This has been deprecated in favour of +// GetDiscoveryAPIForAPIVersionAndResource func (f *Fixture) GetAPIForAPIVersionAndResource( apiversion string, resource string, @@ -234,6 +237,32 @@ func (f *Fixture) GetAPIForAPIVersionAndResource( ) } +// GetDiscoveryAPIForAPIVersionAndResource returns the discovered api based +// on the provided api version & resource +func (f *Fixture) GetDiscoveryAPIForAPIVersionAndResource( + apiversion string, + resource string, +) *dynamicdiscovery.APIResource { + return f.apiDiscovery. + GetAPIForAPIVersionAndResource( + apiversion, + resource, + ) +} + +// GetDiscoveryAPIForAPIVersionAndKind returns the discovered api based +// on the provided api version & resource +func (f *Fixture) GetDiscoveryAPIForAPIVersionAndKind( + apiversion string, + kind string, +) *dynamicdiscovery.APIResource { + return f.apiDiscovery. + GetAPIForAPIVersionAndKind( + apiversion, + kind, + ) +} + // GetAPIResourcesForKind returns the list of discoverd api resources // based on the provided kind func (f *Fixture) GetAPIResourcesForKind(kind string) []*metav1.APIResource { diff --git a/pkg/recipe/list.go b/pkg/recipe/list.go index 3805305..4ae09b0 100644 --- a/pkg/recipe/list.go +++ b/pkg/recipe/list.go @@ -20,9 +20,9 @@ import ( "fmt" "github.com/pkg/errors" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "openebs.io/metac/dynamic/clientset" types "mayadata.io/d-operators/types/recipe" ) @@ -51,40 +51,6 @@ func NewLister(config ListableConfig) *Listable { } } -func (l *Listable) listCRDs() (*types.ListResult, error) { - var crd *v1beta1.CustomResourceDefinition - err := UnstructToTyped(l.List.State, &crd) - if err != nil { - return nil, errors.Wrapf( - err, - "Failed to transform unstruct instance to crd equivalent", - ) - } - // use crd client to list crds - items, err := l.crdClient. - CustomResourceDefinitions(). - List(metav1.ListOptions{ - LabelSelector: labels.Set( - l.List.State.GetLabels(), - ).String(), - }) - if err != nil { - return nil, errors.Wrapf( - err, - "Failed to list crds", - ) - } - return &types.ListResult{ - Phase: types.ListStatusPassed, - Message: fmt.Sprintf( - "List CRD: Kind %s: APIVersion %s", - crd.Spec.Names.Singular, - crd.Spec.Group+"/"+crd.Spec.Version, - ), - V1Beta1CRDItems: items, - }, nil -} - func (l *Listable) listResources() (*types.ListResult, error) { var message = fmt.Sprintf( "List resources with %s / %s: GVK %s", @@ -92,9 +58,23 @@ func (l *Listable) listResources() (*types.ListResult, error) { l.List.State.GetName(), l.List.State.GroupVersionKind(), ) - client, err := l.GetClientForAPIVersionAndKind( - l.List.State.GetAPIVersion(), - l.List.State.GetKind(), + + var client *clientset.ResourceClient + var err error + + // --- + // Retry in-case resource client is not yet + // discovered + // --- + err = l.Retry.Waitf( + func() (bool, error) { + client, err = l.GetClientForAPIVersionAndKind( + l.List.State.GetAPIVersion(), + l.List.State.GetKind(), + ) + return err == nil, err + }, + message, ) if err != nil { return nil, errors.Wrapf( @@ -125,9 +105,5 @@ func (l *Listable) listResources() (*types.ListResult, error) { // Run executes applying the desired state against the // cluster func (l *Listable) Run() (*types.ListResult, error) { - if l.List.State.GetKind() == "CustomResourceDefinition" { - // list CRDs - return l.listCRDs() - } return l.listResources() } diff --git a/pkg/recipe/list_int_crds_test.go b/pkg/recipe/list_int_crds_test.go new file mode 100644 index 0000000..7516070 --- /dev/null +++ b/pkg/recipe/list_int_crds_test.go @@ -0,0 +1,349 @@ +// +build integration + +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + types "mayadata.io/d-operators/types/recipe" +) + +func TestListCRDsRun(t *testing.T) { + tasks := []types.Task{ + { + Name: "create-crd-v1", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "vonelists.openebs.io", + "labels": map[string]interface{}{ + "list-crd-v1-testing": "true", + }, + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "scope": "Namespaced", + "names": map[string]interface{}{ + "kind": "VoneList", + "listKind": "VoneListList", + "plural": "vonelists", + "singular": "vonelist", + "shortNames": []interface{}{ + "vone", + }, + }, + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + "served": true, + "storage": true, + "subresources": map[string]interface{}{ + "status": map[string]interface{}{}, + }, + "schema": map[string]interface{}{ + "openAPIV3Schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "apiVersion": map[string]interface{}{ + "type": "string", + }, + "kind": map[string]interface{}{ + "type": "string", + }, + "metadata": map[string]interface{}{ + "type": "object", + }, + "spec": map[string]interface{}{ + "description": "Specification of the mayastor pool.", + "type": "object", + "required": []interface{}{ + "node", + "disks", + }, + "properties": map[string]interface{}{ + "node": map[string]interface{}{ + "description": "Name of the k8s node where the storage pool is located.", + "type": "string", + }, + "disks": map[string]interface{}{ + "description": "Disk devices (paths or URIs) that should be used for the pool.", + "type": "array", + "items": map[string]interface{}{ + "type": "string", + }, + }, + }, + }, + "status": map[string]interface{}{ + "description": "Status part updated by the pool controller.", + "type": "object", + "properties": map[string]interface{}{ + "state": map[string]interface{}{ + "description": "Pool state.", + "type": "string", + }, + "reason": map[string]interface{}{ + "description": "Reason for the pool state value if applicable.", + "type": "string", + }, + "disks": map[string]interface{}{ + "description": "Disk device URIs that are actually used for the pool.", + "type": "array", + "items": map[string]interface{}{ + "type": "string", + }, + }, + "capacity": map[string]interface{}{ + "description": "Capacity of the pool in bytes.", + "type": "integer", + "format": "int64", + "minimum": int64(0), + }, + "used": map[string]interface{}{ + "description": "How many bytes are used in the pool.", + "type": "integer", + "format": "int64", + "minimum": int64(0), + }, + }, + }, + }, + }, + }, + "additionalPrinterColumns": []interface{}{ + map[string]interface{}{ + "name": "Node", + "type": "string", + "description": "Node where the storage pool is located", + "jsonPath": ".spec.node", + }, + map[string]interface{}{ + "name": "State", + "type": "string", + "description": "State of the storage pool", + "jsonPath": ".status.state", + }, + map[string]interface{}{ + "name": "Age", + "type": "date", + "jsonPath": ".metadata.creationTimestamp", + }, + }, + }, + }, + }, + }, + }, + IgnoreDiscovery: true, + }, + }, + { + Name: "create-crd-v1beta1", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1beta1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "name": "betalists.openebs.io", + "labels": map[string]interface{}{ + "list-crd-v1beta1-testing": "true", + }, + }, + "spec": map[string]interface{}{ + "group": "openebs.io", + "scope": "Namespaced", + "names": map[string]interface{}{ + "kind": "BetaList", + "listKind": "BetaListList", + "plural": "betalists", + "singular": "betalist", + "shortNames": []interface{}{ + "bl", + }, + }, + "version": "v1alpha1", + "versions": []interface{}{ + map[string]interface{}{ + "name": "v1alpha1", + "served": true, + "storage": true, + }, + }, + }, + }, + }, + IgnoreDiscovery: true, + }, + }, + } + recipe := types.Recipe{ + Spec: types.RecipeSpec{ + Tasks: tasks, + }, + } + runner, err := NewNonCustomResourceRunnerWithOptions( + "list-crds-integration-testing", + recipe, + NonCustomResourceRunnerOption{ + SingleTry: true, // Should Work Since Discovery is False + Teardown: false, // Should Not Teardown For Further List Ops + }, + ) + if err != nil { + t.Fatalf( + "Failed to create kubernetes runner: %v", + err, + ) + } + result, err := runner.RunWithoutLocking() + if err != nil { + t.Fatalf("Error while testing: %v: %s", err, result) + } + if !(result.Phase == types.RecipeStatusCompleted || + result.Phase == types.RecipeStatusPassed) { + t.Fatalf("Test failed: %s", result) + } + + // ---- + // Test Lister feature for CRD v1beta + // ---- + br, err := NewDefaultBaseRunner("list-crds-int-testing") + if err != nil { + t.Fatalf("Failed to create base runner: %v", err) + } + l1 := NewLister(ListableConfig{ + BaseRunner: *br, + List: &types.List{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1beta1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "list-crd-v1beta1-testing": "true", + }, + }, + }, + }, + }, + }) + res, err := l1.Run() + if err != nil { + t.Fatalf("Failed to execute lister: %v", err) + } + if res.Phase != types.ListStatusPassed { + t.Fatalf("Lister execution resulted in error: %s", res) + } + if res.Items == nil || len(res.Items.Items) != 1 { + t.Fatalf( + "Invalid list count: Got %d: Want 1", len(res.Items.Items), + ) + } + + // ---- + // Test Lister Feature For CRD v1beta Instance + // ---- + l11 := NewLister(ListableConfig{ + BaseRunner: *br, + List: &types.List{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "openebs.io/v1alpha1", + "kind": "BetaList", + }, + }, + }, + }) + res, err = l11.Run() + if err != nil { + t.Fatalf("Failed to execute lister: %v", err) + } + if res.Phase != types.ListStatusPassed { + t.Fatalf("Lister execution resulted in error: %s", res) + } + if res.Items == nil || len(res.Items.Items) != 0 { + t.Fatalf( + "Invalid list count: Got %d: Want 0", len(res.Items.Items), + ) + } + + // ---- + // Test Lister feature for CRD v1 + // ---- + l2 := NewLister(ListableConfig{ + BaseRunner: *br, + List: &types.List{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "list-crd-v1-testing": "true", + }, + }, + }, + }, + }, + }) + res, err = l2.Run() + if err != nil { + t.Fatalf("Failed to execute lister: %v", err) + } + if res.Phase != types.ListStatusPassed { + t.Fatalf("Lister execution resulted in error: %s", res) + } + if res.Items == nil || len(res.Items.Items) != 1 { + t.Fatalf( + "Invalid list count: Got %d: Want 1", len(res.Items.Items), + ) + } + + // ---- + // Test Lister Feature For CRD v1 Instance + // ---- + l22 := NewLister(ListableConfig{ + BaseRunner: *br, + List: &types.List{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "openebs.io/v1alpha1", + "kind": "VoneList", + }, + }, + }, + }) + res, err = l22.Run() + if err != nil { + t.Fatalf("Failed to execute lister: %v", err) + } + if res.Phase != types.ListStatusPassed { + t.Fatalf("Lister execution resulted in error: %s", res) + } + if res.Items == nil || len(res.Items.Items) != 0 { + t.Fatalf( + "Invalid list count: Got %d: Want 0", len(res.Items.Items), + ) + } +} diff --git a/pkg/recipe/list_int_items_test.go b/pkg/recipe/list_int_items_test.go new file mode 100644 index 0000000..493c319 --- /dev/null +++ b/pkg/recipe/list_int_items_test.go @@ -0,0 +1,156 @@ +// +build integration + +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "mayadata.io/d-operators/common/pointer" + types "mayadata.io/d-operators/types/recipe" +) + +func TestListItemsRun(t *testing.T) { + tasks := []types.Task{ + { + Name: "create-ns", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "list-items-integration-testing", + }, + }, + }, + }, + }, + { + Name: "create-configmap", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "cm-one", + "namespace": "list-items-integration-testing", + }, + }, + }, + }, + }, + { + Name: "create-configmap-2", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "cm-two", + "namespace": "list-items-integration-testing", + }, + }, + }, + }, + }, + { + Name: "assert-configmap-list", + Assert: &types.Assert{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "namespace": "list-items-integration-testing", + }, + }, + }, + StateCheck: &types.StateCheck{ + Operator: types.StateCheckOperatorListCountEquals, + Count: pointer.Int(2), + }, + }, + }, + } + recipe := types.Recipe{ + Spec: types.RecipeSpec{ + Tasks: tasks, + }, + } + runner, err := NewNonCustomResourceRunnerWithOptions( + "list-simple-integration-testing", + recipe, + NonCustomResourceRunnerOption{ + SingleTry: true, + Teardown: false, + }, + ) + if err != nil { + t.Fatalf( + "Failed to create kubernetes runner: %v", + err, + ) + } + result, err := runner.RunWithoutLocking() + if err != nil { + t.Fatalf("Error while testing: %v: %s", err, result) + } + if !(result.Phase == types.RecipeStatusCompleted || + result.Phase == types.RecipeStatusPassed) { + t.Fatalf("Test failed: %s", result) + } + + // ---- + // Test Lister feature + // ---- + br, err := NewDefaultBaseRunner("list-items-int-testing") + if err != nil { + t.Fatalf("Failed to create base runner: %v", err) + } + l := NewLister(ListableConfig{ + BaseRunner: *br, + List: &types.List{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "namespace": "list-items-integration-testing", + }, + }, + }, + }, + }) + res, err := l.Run() + if err != nil { + t.Fatalf("Failed to execute lister: %v", err) + } + if res.Phase != types.ListStatusPassed { + t.Fatalf("Lister execution resulted in error: %s", res) + } + if res.Items == nil || len(res.Items.Items) != 2 { + t.Fatalf( + "Lister execution resulted in invalid list count: Got %d: Want 2", + len(res.Items.Items), + ) + } +} diff --git a/pkg/recipe/labeling_int_test.go b/pkg/recipe/recipe_int_labeling_test.go similarity index 100% rename from pkg/recipe/labeling_int_test.go rename to pkg/recipe/recipe_int_labeling_test.go diff --git a/pkg/recipe/recipe_int_list_test.go b/pkg/recipe/recipe_int_list_test.go new file mode 100644 index 0000000..5d96a03 --- /dev/null +++ b/pkg/recipe/recipe_int_list_test.go @@ -0,0 +1,150 @@ +// +build integration + +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "mayadata.io/d-operators/common/pointer" + types "mayadata.io/d-operators/types/recipe" +) + +func TestListSimpleRun(t *testing.T) { + tasks := []types.Task{ + { + Name: "create-ns", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "list-simple-integration-testing", + }, + }, + }, + }, + }, + { + Name: "create-configmap", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "cm-one", + "namespace": "list-simple-integration-testing", + }, + }, + }, + }, + }, + { + Name: "create-configmap-2", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "cm-two", + "namespace": "list-simple-integration-testing", + }, + }, + }, + }, + }, + { + Name: "assert-ns", + Assert: &types.Assert{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "list-simple-integration-testing", + }, + }, + }, + }, + }, + { + Name: "assert-configmap", + Assert: &types.Assert{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "cm-one", + "namespace": "list-simple-integration-testing", + }, + }, + }, + }, + }, + { + Name: "assert-configmap-list", + Assert: &types.Assert{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "namespace": "list-simple-integration-testing", + }, + }, + }, + StateCheck: &types.StateCheck{ + Operator: types.StateCheckOperatorListCountEquals, + Count: pointer.Int(2), + }, + }, + }, + } + recipe := types.Recipe{ + Spec: types.RecipeSpec{ + Tasks: tasks, + }, + } + runner, err := NewNonCustomResourceRunnerWithOptions( + "list-simple-integration-testing", + recipe, + NonCustomResourceRunnerOption{ + SingleTry: true, + Teardown: true, + }, + ) + if err != nil { + t.Fatalf( + "Failed to create kubernetes runner: %v", + err, + ) + } + result, err := runner.RunWithoutLocking() + if err != nil { + t.Fatalf("Error while testing: %v: %s", err, result) + } + if !(result.Phase == types.RecipeStatusCompleted || + result.Phase == types.RecipeStatusPassed) { + t.Fatalf("Test failed: %s", result) + } +} diff --git a/pkg/recipe/recipe_int_simple_test.go b/pkg/recipe/recipe_int_simple_test.go new file mode 100644 index 0000000..015ffa1 --- /dev/null +++ b/pkg/recipe/recipe_int_simple_test.go @@ -0,0 +1,163 @@ +// +build integration + +/* +Copyright 2020 The MayaData Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recipe + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + types "mayadata.io/d-operators/types/recipe" +) + +func TestRecipeSimpleRun(t *testing.T) { + tasks := []types.Task{ + { + Name: "create-ns", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "recipe-integration-testing-simple", + }, + }, + }, + }, + }, + { + Name: "create-configmap", + Create: &types.Create{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "cm-one", + "namespace": "recipe-integration-testing-simple", + "labels": map[string]interface{}{ + "common": "true", + "cm-one": "true", + }, + }, + }, + }, + }, + }, + { + Name: "apply-ns", + Apply: &types.Apply{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "recipe-integration-testing-simple", + "labels": map[string]interface{}{ + "new": "new", + }, + }, + }, + }, + }, + }, + { + Name: "apply-configmap", + Apply: &types.Apply{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "cm-one", + "namespace": "recipe-integration-testing-simple", + "labels": map[string]interface{}{ + "cm-new": "new", + }, + }, + }, + }, + }, + }, + { + Name: "assert-ns", + Assert: &types.Assert{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "recipe-integration-testing-simple", + "labels": map[string]interface{}{ + "new": "new", + }, + }, + }, + }, + }, + }, + { + Name: "assert-configmap", + Assert: &types.Assert{ + State: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "cm-one", + "namespace": "recipe-integration-testing-simple", + "labels": map[string]interface{}{ + "common": "true", + "cm-one": "true", + "cm-new": "new", + }, + }, + }, + }, + }, + }, + } + recipe := types.Recipe{ + Spec: types.RecipeSpec{ + Tasks: tasks, + }, + } + runner, err := NewNonCustomResourceRunnerWithOptions( + "integration-testing-simple-recipe", + recipe, + NonCustomResourceRunnerOption{ + SingleTry: true, + Teardown: true, + }, + ) + if err != nil { + t.Fatalf( + "Failed to create kubernetes runner: %v", + err, + ) + } + result, err := runner.RunWithoutLocking() + if err != nil { + t.Fatalf("Error while testing: %v: %s", err, result) + } + if !(result.Phase == types.RecipeStatusCompleted || + result.Phase == types.RecipeStatusPassed) { + t.Fatalf("Test failed: %s", result) + } +} diff --git a/test/integration/suite.sh b/test/integration/suite.sh index 83b40da..07e127b 100755 --- a/test/integration/suite.sh +++ b/test/integration/suite.sh @@ -82,27 +82,34 @@ k3s kubectl get node echo -e "\n Apply integration manifests to K3s cluster" k3s kubectl apply -f it.yaml -echo -e "\n Retry 50 times until integration test job gets completed" +echo -e "\n Will retry 50 times until integration test job gets completed" + +echo -e "\n Start Time" date +echo -e "\n" + phase="" for i in {1..50} do succeeded=$(k3s kubectl get job inference -n dit -o=jsonpath='{.status.succeeded}') failed=$(k3s kubectl get job inference -n dit -o=jsonpath='{.status.failed}') - echo -e "Attempt $i: Inference status: status.succeeded='$succeeded'" - echo -e "Attempt $i: Inference status: status.failed='$failed'" + echo -e "Attempt $i: status.succeeded='$succeeded' status.failed='$failed'" if [[ "$failed" == "1" ]]; then break # Abandon this loop since job has failed fi + if [[ "$succeeded" != "1" ]]; then - sleep 10 # Sleep & retry since experiment is in-progress + sleep 15 # Sleep & retry since experiment is in-progress else break # Abandon this loop since succeeded is set fi done + +echo -e "\n End Time" date +echo -e "\n" echo -e "\n Display status of inference job" k3s kubectl get job inference -n dit -ojson | jq .status || true @@ -110,7 +117,7 @@ k3s kubectl get job inference -n dit -ojson | jq .status || true echo -e "\n Display test logs & coverage" k3s kubectl -n dit logs -ljob-name=inference --tail=-1 || true -if [[ "$succeeded" != "1" ]]; then +if [ "$succeeded" != "1" ] || [ "$failed" == "1" ]; then echo "" echo "--------------------------" echo -e "++ Integration test suite failed:" diff --git a/types/recipe/create.go b/types/recipe/create.go index 1a724a0..968d39e 100644 --- a/types/recipe/create.go +++ b/types/recipe/create.go @@ -66,7 +66,7 @@ const ( CreateStatusFailed CreateStatusPhase = "Failed" ) -// ToTaskStatusPhase transforms CreateStatusPhase to TestResultPhase +// ToTaskStatusPhase transforms CreateStatusPhase to TaskStatusPhase func (phase CreateStatusPhase) ToTaskStatusPhase() TaskStatusPhase { switch phase { case CreateStatusPassed: @@ -80,6 +80,20 @@ func (phase CreateStatusPhase) ToTaskStatusPhase() TaskStatusPhase { } } +// ToApplyStatusPhase transforms CreateStatusPhase to ApplyStatusPhase +func (phase CreateStatusPhase) ToApplyStatusPhase() ApplyStatusPhase { + switch phase { + case CreateStatusPassed: + return ApplyStatusPassed + case CreateStatusFailed: + return ApplyStatusFailed + case CreateStatusWarning: + return ApplyStatusWarning + default: + return "" + } +} + // CreateResult holds the result of the create operation type CreateResult struct { Phase CreateStatusPhase `json:"phase"` diff --git a/types/recipe/list.go b/types/recipe/list.go index fc0a810..85a0291 100644 --- a/types/recipe/list.go +++ b/types/recipe/list.go @@ -82,3 +82,16 @@ type ListResult struct { V1Beta1CRDItems *v1beta1.CustomResourceDefinitionList `json:"v1b1CRDs,omitempty"` Items *unstructured.UnstructuredList `json:"items,omitempty"` } + +// String implements the Stringer interface +func (l ListResult) String() string { + raw, err := json.MarshalIndent( + l, + " ", + ".", + ) + if err != nil { + panic(err) + } + return string(raw) +}