feat(kubernetes)!: wait logic kubernetes version support

docs/reference/domains/kubernetes-domain.md

@@ -29,18 +29,19 @@ domain:
           base64:                       # Optional - Boolean whether field is base64 encoded
 ```
 
-> [!Tip]
-> Lula supports eventual-consistency through use of an optional `wait` field in the `kubernetes-spec`. 
+Lula supports eventual-consistency through use of an optional `wait` field in the `kubernetes-spec`. This parameter supports waiting for a specified resource to be `Ready` in the cluster. This may be particularly useful if evaluating the status of a selected resource or evaluating the children of a specified resource.
 
 ```yaml
 domain:
   type: kubernetes
   kubernetes-spec:
     wait:                               # Optional - Group of resources to read from Kubernetes
-      condition: Ready                  # ...
-      kind: pod/test-pod-wait           # ...
-      namespace: validation-test        # ...
-      timeout: 30s                      # ...
+      group:                            # Optional - Empty or "" for core group
+      version: v1                       # Required - Version of resource
+      resource: pods                    # Required - Resource type (API-recognized type, not Kind)Required - Resource type (API-recognized type, not Kind)
+      name: test-pod-wait               # Required - Name of the resource to wait for
+      namespace: validation-test        # Optional - For namespaced resources
+      timeout: 30s                      # Optional - Defaults to 30s
     resources:
     - name: podsvt
       resource-rule:
@@ -50,6 +51,9 @@ domain:
         namespaces: [validation-test]
 ```
 
+> [!Tip]
+> Both `resources` and `wait` use the Group, Version, Resource constructs to identify the resource to be evaluated. To identify those using `kubectl`, executing `kubectl explain <resource/kind/short name>` will provide the Group and Version, the `resource` field is the API-recognized type and can be confirmed by consulting the list provided by `kubectl api-resources`.
+
 ### Resource Creation
 
 The Kubernetes domain also supports creating, reading, and destroying test resources in the cluster. This feature should be used with caution since it's writing to the cluster and ideally should be implemented on separate namespaces to make any erroneous outcomes easier to mitigate.

go.mod

@@ -66,6 +66,7 @@ require (
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/daviddengcn/go-colortext v1.0.0 // indirect
+	github.com/defenseunicorns/pkg/kubernetes v0.3.0 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.2 // indirect
@@ -193,6 +194,7 @@ require (
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/metrics v0.31.1 // indirect
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
+	sigs.k8s.io/cli-utils v0.36.0 // indirect
 	muzzammil.xyz/jsonc v1.0.0 // indirect
 	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
 	sigs.k8s.io/controller-runtime v0.18.2 // indirect

go.sum

@@ -121,6 +121,8 @@ github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX
 github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c=
 github.com/defenseunicorns/go-oscal v0.6.0 h1:eflEKfk7edu4L4kWf6aNQpS94ljfGP8lgWpsPYNtE1Q=
 github.com/defenseunicorns/go-oscal v0.6.0/go.mod h1:UHp2yK9ty2mYJDun7oNhbstCq6SAAwP4YGbw9n7uG6o=
+github.com/defenseunicorns/pkg/kubernetes v0.3.0 h1:f4VSIaUdvn87/dhiZvRbUfHhcHa8bKia6aU0WcvPbYg=
+github.com/defenseunicorns/pkg/kubernetes v0.3.0/go.mod h1:FsuKQGpPZOnZWifBse7v787+avtIu2lte5LTsaojDkY=
 github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg=
 github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw=
 github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
@@ -683,6 +685,8 @@ k8s.io/metrics v0.31.1 h1:h4I4dakgh/zKflWYAOQhwf0EXaqy8LxAIyE/GBvxqRc=
 k8s.io/metrics v0.31.1/go.mod h1:JuH1S9tJiH9q1VCY0yzSCawi7kzNLsDzlWDJN4xR+iA=
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/cli-utils v0.36.0 h1:k7GM6LmIMydtvM6Ad91XuqKk0QEVL9bVbaiX1uvWIrA=
+sigs.k8s.io/cli-utils v0.36.0/go.mod h1:uCFC3BPXB3xHFQyKkWUlTrncVDCKzbdDfqZqRTCrk24=
 muzzammil.xyz/jsonc v1.0.0 h1:B6kaT3wHueZ87mPz3q1nFuM1BlL32IG0wcq0/uOsQ18=
 muzzammil.xyz/jsonc v1.0.0/go.mod h1:rFv8tUUKe+QLh7v02BhfxXEf4ZHhYD7unR93HL/1Uvo=
 olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ=

src/cmd/dev/common.go

@@ -83,7 +83,7 @@ func ReadValidation(cmd *cobra.Command, spinner *message.Spinner, path string, t
 }
 
 // RunSingleValidation runs a single validation
-func RunSingleValidation(validationBytes []byte, opts ...types.LulaValidationOption) (lulaValidation types.LulaValidation, err error) {
+func RunSingleValidation(ctx context.Context, validationBytes []byte, opts ...types.LulaValidationOption) (lulaValidation types.LulaValidation, err error) {
 	var validation common.Validation
 
 	err = yaml.Unmarshal(validationBytes, &validation)
@@ -96,7 +96,7 @@ func RunSingleValidation(validationBytes []byte, opts ...types.LulaValidationOpt
 		return lulaValidation, err
 	}
 
-	err = lulaValidation.Validate(context.Background(), opts...)
+	err = lulaValidation.Validate(ctx, opts...)
 	if err != nil {
 		return lulaValidation, err
 	}

src/cmd/dev/get-resources.go

@@ -72,7 +72,7 @@ func init() {
 }
 
 func DevGetResources(ctx context.Context, validationBytes []byte, spinner *message.Spinner) (types.DomainResources, error) {
-	lulaValidation, err := RunSingleValidation(
+	lulaValidation, err := RunSingleValidation(ctx,
 		validationBytes,
 		types.ExecutionAllowed(getResourcesOpts.ConfirmExecution),
 		types.Interactive(RunInteractively),

src/cmd/dev/validate.go

@@ -132,7 +132,7 @@ func DevValidate(ctx context.Context, validationBytes []byte, resourcesBytes []b
 		}
 	}
 
-	lulaValidation, err = RunSingleValidation(
+	lulaValidation, err = RunSingleValidation(ctx,
 		validationBytes,
 		types.WithStaticResources(resources),
 		types.ExecutionAllowed(validateOpts.ConfirmExecution),

src/pkg/common/common.go

@@ -140,13 +140,13 @@ func SetCwdToFileDir(dirPath string) (resetFunc func(), err error) {
 }
 
 // Get the domain and providers
-func GetDomain(domain *Domain, ctx context.Context) (types.Domain, error) {
+func GetDomain(domain *Domain) (types.Domain, error) {
 	if domain == nil {
 		return nil, fmt.Errorf("domain is nil")
 	}
 	switch domain.Type {
 	case "kubernetes":
-		return kube.CreateKubernetesDomain(ctx, domain.KubernetesSpec)
+		return kube.CreateKubernetesDomain(domain.KubernetesSpec)
 	case "api":
 		return api.CreateApiDomain(domain.ApiSpec)
 	case "file":

src/pkg/common/common_test.go

@@ -116,11 +116,9 @@ func TestGetDomain(t *testing.T) {
 		},
 	}
 
-	ctx := context.Background()
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result, err := common.GetDomain(&tt.domain, ctx)
+			result, err := common.GetDomain(&tt.domain)
 			if (err != nil) != tt.expectedErr {
 				t.Fatalf("expected error: %v, got: %v", tt.expectedErr, err)
 			}

src/pkg/common/schemas/validation.json

@@ -181,17 +181,21 @@
                 "wait": {
                     "type": "object",
                     "properties": {
-                        "condition": {
+                        "name": {
                             "type": "string",
-                            "description": "Condition to wait for ie. 'Ready'"
+                            "description": "Name of the resource to wait for"
                         },
-                        "jsonpath": {
+                        "group": {
                             "type": "string",
-                            "description": "Jsonpath specifier of where to find the condition from the top level object"
+                            "description": "Empty or \"\" for core group"
                         },
-                        "kind": {
+                        "version": {
                             "type": "string",
-                            "description": "Kind of resource to wait for"
+                            "description": "Version of resource"
+                        },
+                        "resource": {
+                            "type": "string",
+                            "description": "Resource type (API-recognized type, not Kind)"
                         },
                         "namespace": {
                             "type": "string",
@@ -201,7 +205,12 @@
                             "type": "string",
                             "description": "Timeout for the wait"
                         }
-                    }
+                    },
+                    "required": [
+                        "name",
+                        "version",
+                        "resource"
+                    ]
                 }
             },
             "anyOf": [

src/pkg/common/types.go

@@ -150,7 +150,7 @@ func (validation *Validation) ToLulaValidation(uuid string) (lulaValidation type
 	// TODO: Is there a better location for context?
 	ctx := context.Background()
 
-	domain, err := GetDomain(validation.Domain, ctx)
+	domain, err := GetDomain(validation.Domain)
 	if domain == nil {
 		return lulaValidation, fmt.Errorf("%w: %s", ErrInvalidDomain, validation.Domain.Type)
 	} else if err != nil {

src/pkg/domains/kubernetes/cluster.go

@@ -0,0 +1,95 @@
+package kube
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+
+	pkgkubernetes "github.com/defenseunicorns/pkg/kubernetes"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/cli-utils/pkg/kstatus/watcher"
+	"sigs.k8s.io/e2e-framework/klient"
+)
+
+var (
+	clusterConnectOnce  sync.Once
+	globalCluster       *Cluster
+	globalConnectionErr error
+)
+
+type Cluster struct {
+	clientset     kubernetes.Interface
+	kclient       klient.Client
+	watcher       watcher.StatusWatcher
+	dynamicClient *dynamic.DynamicClient
+}
+
+func GetCluster() (*Cluster, error) {
+	clusterConnectOnce.Do(func() {
+		globalCluster, globalConnectionErr = New()
+	})
+	if globalConnectionErr != nil {
+		return nil, globalConnectionErr
+	}
+	return globalCluster, globalConnectionErr
+}
+
+func New() (*Cluster, error) {
+	clusterErr := errors.New("unable to connect to the cluster")
+	clientset, config, err := pkgkubernetes.ClientAndConfig()
+	if err != nil {
+		return nil, errors.Join(clusterErr, err)
+	}
+
+	watcher, err := pkgkubernetes.WatcherForConfig(config)
+	if err != nil {
+		return nil, errors.Join(clusterErr, err)
+	}
+
+	kclient, err := klient.New(config)
+	if err != nil {
+		return nil, errors.Join(clusterErr, err)
+	}
+
+	dynamicClient := dynamic.NewForConfigOrDie(config)
+
+	// Ensure no errors were returned to validate cluster connection.
+	_, err = clientset.Discovery().ServerVersion()
+	if err != nil {
+		return nil, errors.Join(clusterErr, err)
+	}
+
+	return &Cluster{
+		clientset:     clientset,
+		kclient:       kclient,
+		watcher:       watcher,
+		dynamicClient: dynamicClient,
+	}, nil
+}
+
+func (c *Cluster) validateAndGetGVR(group, version, resource string) (*metav1.APIResource, error) {
+	// Create a discovery client
+	discoveryClient := c.clientset.Discovery()
+
+	// Get a list of all API resources for the given group version
+	gv := schema.GroupVersion{
+		Group:   group,
+		Version: version,
+	}
+	resourceList, err := discoveryClient.ServerResourcesForGroupVersion(gv.String())
+	if err != nil {
+		return nil, err
+	}
+
+	// Search for the specified resource in the list
+	for _, apiResource := range resourceList.APIResources {
+		if apiResource.Name == resource {
+			return &apiResource, nil
+		}
+	}
+
+	return nil, fmt.Errorf("resource %s not found in group %s version %s", resource, group, version)
+}

src/pkg/domains/kubernetes/create.go

@@ -23,19 +23,13 @@ import (
 )
 
 // CreateE2E() creates the test resources, reads status, and destroys them
-func CreateE2E(ctx context.Context, resources []CreateResource) (map[string]interface{}, error) {
+func CreateE2E(ctx context.Context, cluster *Cluster, resources []CreateResource) (map[string]interface{}, error) {
 	collections := make(map[string]interface{}, len(resources))
 	namespaces := make([]string, 0)
 	var errList []string
 
-	// Set up the clients
-	config, err := connect()
-	if err != nil {
-		return nil, fmt.Errorf("failed to connect to k8s cluster: %w", err)
-	}
-	client, err := klient.New(config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create e2e client: %w", err)
+	if cluster == nil {
+		return nil, fmt.Errorf("cluster is nil")
 	}
 
 	// Create the resources, collect the outcome
@@ -44,7 +38,7 @@ func CreateE2E(ctx context.Context, resources []CreateResource) (map[string]inte
 		var err error
 		// Create namespace if specified
 		if resource.Namespace != "" {
-			new, err := createNamespace(ctx, client, resource.Namespace)
+			new, err := createNamespace(ctx, cluster.kclient, resource.Namespace)
 			if err != nil {
 				message.Debugf("error creating namespace %s: %v", resource.Namespace, err)
 				errList = append(errList, err.Error())
@@ -58,13 +52,13 @@ func CreateE2E(ctx context.Context, resources []CreateResource) (map[string]inte
 		// TODO: Allow both Manifest and File to be specified?
 		// Want to catch any errors and proceed in case resources have already been created
 		if resource.Manifest != "" {
-			collection, err = CreateFromManifest(ctx, client, []byte(resource.Manifest))
+			collection, err = CreateFromManifest(ctx, cluster.kclient, []byte(resource.Manifest))
 			if err != nil {
 				message.Debugf("error creating resource from manifest: %v", err)
 				errList = append(errList, err.Error())
 			}
 		} else if resource.File != "" {
-			collection, err = CreateFromFile(ctx, client, resource.File)
+			collection, err = CreateFromFile(ctx, cluster.kclient, resource.File)
 			if err != nil {
 				message.Debugf("error creating resource from file: %v", err)
 				errList = append(errList, err.Error())
@@ -77,7 +71,7 @@ func CreateE2E(ctx context.Context, resources []CreateResource) (map[string]inte
 	}
 
 	// Destroy the resources
-	if err = DestroyAllResources(ctx, client, collections, namespaces); err != nil {
+	if err := DestroyAllResources(ctx, cluster.kclient, collections, namespaces); err != nil {
 		// If a resource can't be destroyed, return the error (include retry logic??)
 		message.Debugf("error destroying all resources: %v", err)
 		errList = append(errList, err.Error())