diff --git a/go.mod b/go.mod index 0d2915a7..3b8ed819 100644 --- a/go.mod +++ b/go.mod @@ -102,7 +102,7 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.5.1 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/tchap/go-patricia/v2 v2.3.1 // indirect github.com/vladimirvivien/gexe v0.2.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect diff --git a/go.sum b/go.sum index c2e26765..27c287c8 100644 --- a/go.sum +++ b/go.sum @@ -290,17 +290,12 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= -github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes= diff --git a/src/pkg/common/common.go b/src/pkg/common/common.go index e9e797c9..3aa7aa27 100644 --- a/src/pkg/common/common.go +++ b/src/pkg/common/common.go @@ -138,42 +138,31 @@ func SetCwdToFileDir(dirPath string) (resetFunc func(), err error) { } // Get the domain and providers -func GetDomain(domain *Domain, ctx context.Context) types.Domain { +func GetDomain(domain *Domain, ctx context.Context) (types.Domain, error) { if domain == nil { - return nil + return nil, fmt.Errorf("domain is nil") } switch domain.Type { case "kubernetes": - return kube.KubernetesDomain{ - Context: ctx, - Spec: domain.KubernetesSpec, - } + return kube.CreateKubernetesDomain(ctx, domain.KubernetesSpec) case "api": - return api.ApiDomain{ - Spec: domain.ApiSpec, - } + return api.CreateApiDomain(domain.ApiSpec) default: - return nil + return nil, fmt.Errorf("domain is unsupported") } } -func GetProvider(provider *Provider, ctx context.Context) types.Provider { +func GetProvider(provider *Provider, ctx context.Context) (types.Provider, error) { if provider == nil { - return nil + return nil, fmt.Errorf("provider is nil") } switch provider.Type { case "opa": - return opa.OpaProvider{ - Context: ctx, - Spec: provider.OpaSpec, - } + return opa.CreateOpaProvider(ctx, provider.OpaSpec) case "kyverno": - return kyverno.KyvernoProvider{ - Context: ctx, - Spec: provider.KyvernoSpec, - } + return kyverno.CreateKyvernoProvider(ctx, provider.KyvernoSpec) default: - return nil + return nil, fmt.Errorf("provider is unsupported") } } diff --git a/src/pkg/common/common_test.go b/src/pkg/common/common_test.go index 95bb742f..0ffb5f91 100644 --- a/src/pkg/common/common_test.go +++ b/src/pkg/common/common_test.go @@ -12,13 +12,10 @@ import ( kube "github.com/defenseunicorns/lula/src/pkg/domains/kubernetes" "github.com/defenseunicorns/lula/src/pkg/providers/kyverno" "github.com/defenseunicorns/lula/src/pkg/providers/opa" + kjson "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" "sigs.k8s.io/yaml" ) -const validKubernetesPath = "../../test/unit/common/valid-kubernetes-spec.yaml" -const validApiPath = "../../test/unit/common/valid-api-spec.yaml" -const validOpaPath = "../../test/unit/common/valid-opa-spec.yaml" -const validKyvernoPath = "../../test/unit/common/valid-kyverno-spec.yaml" const multiValidationPath = "../../test/e2e/scenarios/remote-validations/multi-validations.yaml" const singleValidationPath = "../../test/e2e/scenarios/remote-validations/validation.opa.yaml" @@ -33,47 +30,88 @@ func loadTestData(t *testing.T, path string) []byte { } func TestGetDomain(t *testing.T) { - validKubernetesBytes := loadTestData(t, validKubernetesPath) - validApiBytes := loadTestData(t, validApiPath) - - var validKubernetes kube.KubernetesSpec - if err := yaml.Unmarshal(validKubernetesBytes, &validKubernetes); err != nil { - t.Fatalf("yaml.Unmarshal failed: %v", err) - } - - var validApi api.ApiSpec - if err := yaml.Unmarshal(validApiBytes, &validApi); err != nil { - t.Fatalf("yaml.Unmarshal failed: %v", err) - } + t.Parallel() - // Define test cases tests := []struct { - name string - domain common.Domain - expected string + name string + domain common.Domain + expectedErr bool + expectedDomain string }{ { - name: "kubernetes domain", + name: "valid kubernetes domain", + domain: common.Domain{ + Type: "kubernetes", + KubernetesSpec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + Name: "podsvt", + ResourceRule: &kube.ResourceRule{ + Version: "v1", + Resource: "pods", + Namespaces: []string{"validation-test"}, + }, + }, + }, + }, + }, + expectedErr: false, + expectedDomain: "kube.KubernetesDomain", + }, + { + name: "invalid kubernetes domain", + domain: common.Domain{ + Type: "kubernetes", + KubernetesSpec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + Name: "podsvt", + ResourceRule: &kube.ResourceRule{ + Version: "v1", + Namespaces: []string{"validation-test"}, + }, + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "valid api domain", domain: common.Domain{ - Type: "kubernetes", - KubernetesSpec: &validKubernetes, + Type: "api", + ApiSpec: &api.ApiSpec{ + Requests: []api.Request{ + { + Name: "local", + URL: "http://localhost", + }, + }, + }, }, - expected: "kube.KubernetesDomain", + expectedErr: false, + expectedDomain: "api.ApiDomain", }, { - name: "api domain", + name: "invalid api domain", domain: common.Domain{ - Type: "api", - ApiSpec: &validApi, + Type: "api", + ApiSpec: &api.ApiSpec{ + Requests: []api.Request{ + { + Name: "local", + }, + }, + }, }, - expected: "api.ApiDomain", + expectedErr: true, }, { - name: "unsupported domain", + name: "invalid type domain", domain: common.Domain{ - Type: "unsupported", + Type: "foo", }, - expected: "nil", + expectedErr: true, }, } @@ -81,9 +119,12 @@ func TestGetDomain(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := common.GetDomain(&tt.domain, ctx) + result, err := common.GetDomain(&tt.domain, ctx) + if (err != nil) != tt.expectedErr { + t.Fatalf("expected error: %v, got: %v", tt.expectedErr, err) + } - switch tt.expected { + switch tt.expectedDomain { case "kube.KubernetesDomain": if _, ok := result.(kube.KubernetesDomain); !ok { t.Errorf("Expected result to be kube.KubernetesDomain, got %T", result) @@ -102,46 +143,60 @@ func TestGetDomain(t *testing.T) { } func TestGetProvider(t *testing.T) { - validOpaBytes := loadTestData(t, validOpaPath) - validKyvernoBytes := loadTestData(t, validKyvernoPath) - - var validOpa opa.OpaSpec - if err := yaml.Unmarshal(validOpaBytes, &validOpa); err != nil { - t.Fatalf("yaml.Unmarshal failed: %v", err) - } - - var validKyverno kyverno.KyvernoSpec - if err := yaml.Unmarshal(validKyvernoBytes, &validKyverno); err != nil { - t.Fatalf("yaml.Unmarshal failed: %v", err) - } + t.Parallel() tests := []struct { - name string - provider common.Provider - expected string + name string + provider common.Provider + expectedErr bool + expectedProvider string }{ { - name: "opa provider", + name: "valid opa provider", + provider: common.Provider{ + Type: "opa", + OpaSpec: &opa.OpaSpec{ + Rego: "package validate\n\ndefault validate = false", + }, + }, + expectedErr: false, + expectedProvider: "opa.OpaProvider", + }, + { + name: "invalid opa provider", provider: common.Provider{ Type: "opa", - OpaSpec: &validOpa, + OpaSpec: &opa.OpaSpec{}, + }, + expectedErr: true, + }, + { + name: "valid kyverno provider", + provider: common.Provider{ + Type: "kyverno", + KyvernoSpec: &kyverno.KyvernoSpec{ + Policy: &kjson.ValidatingPolicy{ + Spec: kjson.ValidatingPolicySpec{}, + }, + }, }, - expected: "opa.OpaProvider", + expectedErr: false, + expectedProvider: "kyverno.KyvernoProvider", }, { - name: "kyverno provider", + name: "invalid kyverno provider", provider: common.Provider{ Type: "kyverno", - KyvernoSpec: &validKyverno, + KyvernoSpec: &kyverno.KyvernoSpec{}, }, - expected: "kyverno.KyvernoProvider", + expectedErr: true, }, { - name: "unsupported provider", + name: "invalid type provider", provider: common.Provider{ - Type: "unsupported", + Type: "foo", }, - expected: "nil", + expectedErr: true, }, } @@ -149,9 +204,12 @@ func TestGetProvider(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := common.GetProvider(&tt.provider, ctx) + result, err := common.GetProvider(&tt.provider, ctx) + if (err != nil) != tt.expectedErr { + t.Fatalf("expected error: %v, got: %v", tt.expectedErr, err) + } - switch tt.expected { + switch tt.expectedProvider { case "opa.OpaProvider": if _, ok := result.(opa.OpaProvider); !ok { t.Errorf("Expected result to be opa.OpaProvider, got %T", result) @@ -374,3 +432,69 @@ func TestReadValidationsFromYaml(t *testing.T) { }) } } + +func TestIsVersionValid(t *testing.T) { + t.Parallel() + tests := []struct { + name string + versionConstraint string + version string + expectedValid bool + expectedErr bool + }{ + { + name: "Valid constraint and version", + versionConstraint: ">= 1.0.0, < 2.0.0", + version: "1.5.0", + expectedValid: true, + expectedErr: false, + }, + { + name: "Valid constraint and version (exact match)", + versionConstraint: "1.0.0", + version: "1.0.0", + expectedValid: true, + expectedErr: false, + }, + { + name: "Invalid version", + versionConstraint: ">= 1.0.0, < 2.0.0", + version: "2.5.0", + expectedValid: false, + expectedErr: false, + }, + { + name: "Invalid constraint syntax", + versionConstraint: ">=> 1.0.0", + version: "1.5.0", + expectedValid: false, + expectedErr: true, + }, + { + name: "Invalid version syntax", + versionConstraint: ">= 1.0.0, < 2.0.0", + version: "invalid", + expectedValid: false, + expectedErr: true, + }, + { + name: "Unset version", + versionConstraint: ">= 1.0.0, < 2.0.0", + version: "unset", + expectedValid: true, + expectedErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + valid, err := common.IsVersionValid(tc.versionConstraint, tc.version) + if (err != nil) != tc.expectedErr { + t.Fatalf("expected error: %v, got: %v", tc.expectedErr, err) + } + if valid != tc.expectedValid { + t.Fatalf("expected valid: %v, got: %v", tc.expectedValid, valid) + } + }) + } +} diff --git a/src/pkg/common/composition/composition_test.go b/src/pkg/common/composition/composition_test.go index 1b924c45..85e15561 100644 --- a/src/pkg/common/composition/composition_test.go +++ b/src/pkg/common/composition/composition_test.go @@ -13,7 +13,9 @@ import ( const ( allRemote = "../../../test/e2e/scenarios/validation-composition/component-definition.yaml" + allRemoteBadHref = "../../../test/e2e/scenarios/validation-composition/component-definition-bad-href.yaml" allLocal = "../../../test/unit/common/composition/component-definition-all-local.yaml" + allLocalBadHref = "../../../test/unit/common/composition/component-definition-all-local-bad-href.yaml" localAndRemote = "../../../test/unit/common/composition/component-definition-local-and-remote.yaml" subComponentDef = "../../../test/unit/common/composition/component-definition-import-compdefs.yaml" compDefMultiImport = "../../../test/unit/common/composition/component-definition-import-multi-compdef.yaml" @@ -30,6 +32,16 @@ func TestComposeFromPath(t *testing.T) { } }) + t.Run("No imports, local validations, bad href", func(t *testing.T) { + model, err := composition.ComposeFromPath(allLocalBadHref) + if err != nil { + t.Fatalf("Error composing component definitions: %v", err) + } + if model == nil { + t.Error("expected the model to be composed") + } + }) + t.Run("No imports, remote validations", func(t *testing.T) { model, err := composition.ComposeFromPath(allRemote) if err != nil { @@ -40,6 +52,16 @@ func TestComposeFromPath(t *testing.T) { } }) + t.Run("No imports, bad remote validations", func(t *testing.T) { + model, err := composition.ComposeFromPath(allRemoteBadHref) + if err != nil { + t.Fatalf("Error composing component definitions: %v", err) + } + if model == nil { + t.Error("expected the model to be composed") + } + }) + t.Run("Errors when file does not exist", func(t *testing.T) { _, err := composition.ComposeFromPath("nonexistent") if err == nil { diff --git a/src/pkg/common/schemas/validation.json b/src/pkg/common/schemas/validation.json index 4e334f4f..2bad3f85 100644 --- a/src/pkg/common/schemas/validation.json +++ b/src/pkg/common/schemas/validation.json @@ -409,13 +409,22 @@ "type": "object", "properties": { "validation": { - "type": "string" + "type": "string", + "description": "optional: variable for validation, must be jsonpath . and resolve to boolean" }, "observations": { - "type": "array", - "items": { - "type": "string" - } + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ], + "description": "optional: any additional observations to include, fields must be jsonpath . and resolve to strings" } } } diff --git a/src/pkg/common/types.go b/src/pkg/common/types.go index 11928f68..161f0bef 100644 --- a/src/pkg/common/types.go +++ b/src/pkg/common/types.go @@ -2,6 +2,7 @@ package common import ( "context" + "errors" "fmt" "strings" @@ -18,6 +19,15 @@ import ( "sigs.k8s.io/yaml" ) +// Define base errors for validations +var ( + ErrInvalidSchema = errors.New("schema is invalid") + ErrInvalidYaml = errors.New("error unmarshaling JSON") + ErrInvalidVersion = errors.New("version is invalid") + ErrInvalidDomain = errors.New("domain is invalid") + ErrInvalidProvider = errors.New("provider is invalid") +) + // Data structures for ingesting validation data type Validation struct { LulaVersion string `json:"lula-version" yaml:"lula-version"` @@ -97,33 +107,38 @@ func (validation *Validation) ToLulaValidation() (lulaValidation types.LulaValid lintResult := validation.Lint() // If the validation is not valid, return the error if oscalValidation.IsNonSchemaValidationError(&lintResult) { - return lulaValidation, oscalValidation.GetNonSchemaError(&lintResult) + return lulaValidation, fmt.Errorf("%w: %v", ErrInvalidSchema, oscalValidation.GetNonSchemaError(&lintResult)) } else if !lintResult.Valid { - return lulaValidation, fmt.Errorf("validation failed: %v", lintResult.Errors) + return lulaValidation, fmt.Errorf("%w: %v", ErrInvalidSchema, lintResult.Errors) } validVersion, versionErr := IsVersionValid(versionConstraint, currentVersion) if versionErr != nil { - return lulaValidation, fmt.Errorf("version error: %s", versionErr.Error()) + return lulaValidation, fmt.Errorf("%w: %s", ErrInvalidVersion, versionErr.Error()) } else if !validVersion { - return lulaValidation, fmt.Errorf("version %s does not meet the constraint %s for this validation", currentVersion, versionConstraint) + return lulaValidation, fmt.Errorf("%w: version %s does not meet the constraint %s for this validation", ErrInvalidVersion, currentVersion, versionConstraint) } // Construct the lulaValidation object // TODO: Is there a better location for context? ctx := context.Background() - provider := GetProvider(validation.Provider, ctx) - if provider == nil { - return lulaValidation, fmt.Errorf("provider %s not found", validation.Provider.Type) - } - lulaValidation.Provider = &provider - domain := GetDomain(validation.Domain, ctx) + domain, err := GetDomain(validation.Domain, ctx) if domain == nil { - return lulaValidation, fmt.Errorf("domain %s not found", validation.Domain.Type) + return lulaValidation, fmt.Errorf("%w: %s", ErrInvalidDomain, validation.Domain.Type) + } else if err != nil { + return lulaValidation, fmt.Errorf("%w: %v", ErrInvalidDomain, err) } lulaValidation.Domain = &domain + provider, err := GetProvider(validation.Provider, ctx) + if provider == nil { + return lulaValidation, fmt.Errorf("%w: %s", ErrInvalidProvider, validation.Provider.Type) + } else if err != nil { + return lulaValidation, fmt.Errorf("%w: %v", ErrInvalidProvider, err) + } + lulaValidation.Provider = &provider + lulaValidation.LulaValidationType = types.DefaultLulaValidationType // TODO: define workflow/purpose for this if validation.Metadata == nil { diff --git a/src/pkg/common/types_test.go b/src/pkg/common/types_test.go new file mode 100644 index 00000000..e5deaa22 --- /dev/null +++ b/src/pkg/common/types_test.go @@ -0,0 +1,189 @@ +package common_test + +import ( + "errors" + "testing" + + "github.com/defenseunicorns/lula/src/config" + "github.com/defenseunicorns/lula/src/pkg/common" +) + +func TestToLulaValidation(t *testing.T) { + t.Parallel() + config.CLIVersion = "1.0.0" // Set the version for testing purposes + + tests := []struct { + name string + inputYaml []byte + expectErr bool + expectedErrType error + }{ + { + name: "Valid validation", + inputYaml: []byte(` +lula-version: "1.0.0" +metadata: + name: "test-valid" +domain: + type: "kubernetes" + kubernetes-spec: + resources: [] +provider: + type: "opa" + opa-spec: + rego: "package validate\n\ndefault validate = false" +`), + expectErr: false, + }, + { + name: "Invalid version", + inputYaml: []byte(` +lula-version: "2.0.0" +metadata: + name: "test-invalid-version" +domain: + type: "kubernetes" + kubernetes-spec: + resources: [] +provider: + type: "opa" + opa-spec: + rego: "package validate\n\ndefault validate = false" +`), + expectErr: true, + expectedErrType: common.ErrInvalidVersion, + }, + { + name: "Invalid schema", + inputYaml: []byte(` +lula-version: "1.0.0" +metadata: {} +`), + expectErr: true, + expectedErrType: common.ErrInvalidSchema, + }, + { + name: "Invalid domain schema, bad type", + inputYaml: []byte(` +lula-version: "1.0.0" +metadata: + name: "test-invalid-domain" +domain: + type: "unknown" +provider: + type: "opa" + opa-spec: + rego: "package validate\n\ndefault validate = false" +`), + expectErr: true, + expectedErrType: common.ErrInvalidSchema, + }, + { + name: "Invalid domain schema, missing spec", + inputYaml: []byte(` +lula-version: "1.0.0" +metadata: + name: "test-invalid-domain" +domain: + type: "kubernetes" +provider: + type: "opa" + opa-spec: + rego: "package validate\n\ndefault validate = false" +`), + expectErr: true, + expectedErrType: common.ErrInvalidSchema, + }, + { + name: "Invalid provider schema, bad type", + inputYaml: []byte(` +lula-version: "1.0.0" +metadata: + name: "test-invalid-provider" +domain: + type: "kubernetes" + kubernetes-spec: + resources: [] +provider: + type: "unknown" +`), + expectErr: true, + expectedErrType: common.ErrInvalidSchema, + }, + { + name: "Invalid provider schema, missing spec", + inputYaml: []byte(` +lula-version: "1.0.0" +metadata: + name: "test-invalid-provider" +domain: + type: "kubernetes" + kubernetes-spec: + resources: [] +provider: + type: "opa" +`), + expectErr: true, + expectedErrType: common.ErrInvalidSchema, + }, + { + name: "Bad kubernetes spec - missing resource", + inputYaml: []byte(` +lula-version: "1.0.0" +metadata: + name: "test-invalid-provider" +domain: + type: "kubernetes" + kubernetes-spec: + resources: + - name: "test" + resource-rule: + version: "test" +provider: + type: "opa" + opa-spec: + rego: "package validate\n\ndefault validate = false" +`), + expectErr: true, + expectedErrType: common.ErrInvalidDomain, + }, + { + name: "Bad opa spec - bad output validation format", + inputYaml: []byte(` +lula-version: "1.0.0" +metadata: + name: "test-invalid-provider" +domain: + type: "kubernetes" + kubernetes-spec: + resources: [] +provider: + type: opa + opa-spec: + rego: "package validate\n\ndefault validate = false" + output: + validation: validate-result +`), + expectErr: true, + expectedErrType: common.ErrInvalidProvider, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var validation common.Validation + err := validation.UnmarshalYaml(tt.inputYaml) + if err != nil { + t.Fatalf("UnmarshalYaml failed: %v", err) + } + + _, err = validation.ToLulaValidation() + if (err != nil) != tt.expectErr { + t.Fatalf("expected error: %v, got: %v", tt.expectErr, err) + } + if (err != nil) && !errors.Is(err, tt.expectedErrType) { + t.Fatalf("expected error type: %v, got: %v", tt.expectedErrType, err) + } + }) + } +} diff --git a/src/pkg/domains/api/types.go b/src/pkg/domains/api/types.go index 92644fa5..bbd28398 100644 --- a/src/pkg/domains/api/types.go +++ b/src/pkg/domains/api/types.go @@ -1,6 +1,8 @@ package api import ( + "fmt" + "github.com/defenseunicorns/lula/src/types" ) @@ -10,6 +12,29 @@ type ApiDomain struct { Spec *ApiSpec `json:"spec,omitempty" yaml:"spec,omitempty"` } +func CreateApiDomain(spec *ApiSpec) (types.Domain, error) { + // Check validity of spec + if spec == nil { + return nil, fmt.Errorf("spec is nil") + } + + if len(spec.Requests) == 0 { + return nil, fmt.Errorf("some requests must be specified") + } + for _, request := range spec.Requests { + if request.Name == "" { + return nil, fmt.Errorf("request name cannot be empty") + } + if request.URL == "" { + return nil, fmt.Errorf("request url cannot be empty") + } + } + + return ApiDomain{ + Spec: spec, + }, nil +} + func (a ApiDomain) GetResources() (types.DomainResources, error) { return MakeRequests(a.Spec.Requests) } diff --git a/src/pkg/domains/api/types_test.go b/src/pkg/domains/api/types_test.go new file mode 100644 index 00000000..64fa9562 --- /dev/null +++ b/src/pkg/domains/api/types_test.go @@ -0,0 +1,74 @@ +package api_test + +import ( + "testing" + + api "github.com/defenseunicorns/lula/src/pkg/domains/api" +) + +func TestCreateApiDomain(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + spec *api.ApiSpec + expectedErr bool + }{ + { + name: "nil spec", + spec: nil, + expectedErr: true, + }, + { + name: "empty requests", + spec: &api.ApiSpec{ + Requests: []api.Request{}, + }, + expectedErr: true, + }, + { + name: "invalid request - no name", + spec: &api.ApiSpec{ + Requests: []api.Request{ + { + URL: "test", + }, + }, + }, + expectedErr: true, + }, + { + name: "invalid request - no url", + spec: &api.ApiSpec{ + Requests: []api.Request{ + { + Name: "test", + }, + }, + }, + expectedErr: true, + }, + { + name: "valid request", + spec: &api.ApiSpec{ + Requests: []api.Request{ + { + Name: "test", + URL: "test", + }, + }, + }, + expectedErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := api.CreateApiDomain(tt.spec) + if (err != nil) != tt.expectedErr { + t.Errorf("CreateApiDomain() error = %v, wantErr %v", err, tt.expectedErr) + return + } + }) + } +} diff --git a/src/pkg/domains/kubernetes/types.go b/src/pkg/domains/kubernetes/types.go index b6f07c13..e276093e 100644 --- a/src/pkg/domains/kubernetes/types.go +++ b/src/pkg/domains/kubernetes/types.go @@ -3,6 +3,7 @@ package kube import ( "context" "errors" + "fmt" "github.com/defenseunicorns/lula/src/types" ) @@ -15,6 +16,77 @@ type KubernetesDomain struct { Spec *KubernetesSpec `json:"spec,omitempty" yaml:"spec,omitempty"` } +func CreateKubernetesDomain(ctx context.Context, spec *KubernetesSpec) (types.Domain, error) { + // Check validity of spec + if spec == nil { + return nil, fmt.Errorf("spec is nil") + } + + if spec.Resources == nil && spec.CreateResources == nil && spec.Wait == nil { + return nil, fmt.Errorf("one of resources, create-resources, or wait must be specified") + } + + if spec.Resources != nil { + for _, resource := range spec.Resources { + if resource.Name == "" { + return nil, fmt.Errorf("resource name cannot be empty") + } + if resource.ResourceRule == nil { + return nil, fmt.Errorf("resource rule cannot be nil") + } + if resource.ResourceRule.Resource == "" { + return nil, fmt.Errorf("resource rule resource cannot be empty") + } + if resource.ResourceRule.Version == "" { + return nil, fmt.Errorf("resource rule version cannot be empty") + } + if resource.ResourceRule.Name != "" && len(resource.ResourceRule.Namespaces) > 1 { + return nil, fmt.Errorf("named resource requested cannot be returned from multiple namespaces") + } + if resource.ResourceRule.Field != nil { + if resource.ResourceRule.Field.Type == "" { + resource.ResourceRule.Field.Type = DefaultFieldType + } + err := resource.ResourceRule.Field.Validate() + if err != nil { + return nil, err + } + if resource.ResourceRule.Name == "" { + return nil, fmt.Errorf("field cannot be specified without resource name") + } + } + } + } + + if spec.Wait != nil { + if spec.Wait.Kind == "" { + return nil, fmt.Errorf("wait kind cannot be empty") + } + if spec.Wait.Condition != "" && spec.Wait.Jsonpath != "" { + return nil, fmt.Errorf("only one of wait.condition or wait.jsonpath can be specified") + } + } + + if spec.CreateResources != nil { + for _, resource := range spec.CreateResources { + if resource.Name == "" { + return nil, fmt.Errorf("resource name cannot be empty") + } + if resource.Manifest == "" && resource.File == "" { + return nil, fmt.Errorf("resource manifest or file must be specified") + } + if resource.Manifest != "" && resource.File != "" { + return nil, fmt.Errorf("only resource manifest or file can be specified") + } + } + } + + return KubernetesDomain{ + Context: ctx, + Spec: spec, + }, nil +} + func (k KubernetesDomain) GetResources() (resources types.DomainResources, err error) { // Evaluate the wait condition if k.Spec.Wait != nil { @@ -24,7 +96,7 @@ func (k KubernetesDomain) GetResources() (resources types.DomainResources, err e } } - // Return both? + // TODO: Return both? if k.Spec.Resources != nil { resources, err = QueryCluster(k.Context, k.Spec.Resources) if err != nil { @@ -42,10 +114,7 @@ func (k KubernetesDomain) GetResources() (resources types.DomainResources, err e func (k KubernetesDomain) IsExecutable() bool { // Domain is only executable if create-resources is not nil - if len(k.Spec.CreateResources) > 0 { - return true - } - return false + return len(k.Spec.CreateResources) > 0 } type KubernetesSpec struct { diff --git a/src/pkg/domains/kubernetes/types_test.go b/src/pkg/domains/kubernetes/types_test.go new file mode 100644 index 00000000..6ea81cbf --- /dev/null +++ b/src/pkg/domains/kubernetes/types_test.go @@ -0,0 +1,265 @@ +package kube_test + +import ( + "context" + "testing" + + kube "github.com/defenseunicorns/lula/src/pkg/domains/kubernetes" +) + +func TestCreateKubernetesDomain(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + spec *kube.KubernetesSpec + expectedErr bool + }{ + { + name: "nil spec", + spec: nil, + expectedErr: true, + }, + { + name: "empty spec", + spec: &kube.KubernetesSpec{}, + expectedErr: true, + }, + { + name: "empty resources", + spec: &kube.KubernetesSpec{ + Resources: []kube.Resource{}, + }, + expectedErr: false, // currently this is not an error to allow space for PLACEHOLDER validations (TODO: implement empty resources error) + }, + { + name: "valid resources", + spec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + Name: "test", + ResourceRule: &kube.ResourceRule{ + Version: "test", + Resource: "test", + }, + }, + }, + }, + expectedErr: false, + }, + { + name: "valid resources with field", + spec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + Name: "test", + ResourceRule: &kube.ResourceRule{ + Name: "test", + Version: "test", + Resource: "test", + Field: &kube.Field{ + Jsonpath: "test", + }, + }, + }, + }, + }, + expectedErr: false, + }, + { + name: "invalid resources - no name", + spec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + ResourceRule: &kube.ResourceRule{ + Version: "test", + Resource: "test", + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "invalid resources - no resource-rule", + spec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + Name: "test", + }, + }, + }, + expectedErr: true, + }, + { + name: "invalid resource-rule, no version", + spec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + Name: "test", + ResourceRule: &kube.ResourceRule{ + Resource: "test", + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "invalid resource-rule, no resource", + spec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + Name: "test", + ResourceRule: &kube.ResourceRule{ + Version: "test", + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "invalid resource-rule, one name, many namespaces", + spec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + Name: "test", + ResourceRule: &kube.ResourceRule{ + Name: "test", + Version: "test", + Resource: "test", + Namespaces: []string{"test-1", "test-2"}, + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "invalid resource-rule, field without name", + spec: &kube.KubernetesSpec{ + Resources: []kube.Resource{ + { + Name: "test", + ResourceRule: &kube.ResourceRule{ + Version: "test", + Resource: "test", + Namespaces: []string{"test-1", "test-2"}, + Field: &kube.Field{ + Jsonpath: "test", + }, + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "empty create-resources", + spec: &kube.KubernetesSpec{ + CreateResources: []kube.CreateResource{}, + }, + expectedErr: false, // currently this is not an error to allow space for PLACEHOLDER validations (TODO: implement empty resources error) + }, + { + name: "valid create-resources with manifest", + spec: &kube.KubernetesSpec{ + CreateResources: []kube.CreateResource{ + { + Name: "test", + Manifest: ` +apiVersion: v1 +kind: Pod +metadata: + name: test +`, + }, + }, + }, + expectedErr: false, + }, + { + name: "valid create-resources with file", + spec: &kube.KubernetesSpec{ + CreateResources: []kube.CreateResource{ + { + Name: "test", + File: "../file/path.yaml", + }, + }, + }, + expectedErr: false, + }, + { + name: "invalid create-resource, missing file and manifest", + spec: &kube.KubernetesSpec{ + CreateResources: []kube.CreateResource{ + { + Name: "test", + }, + }, + }, + expectedErr: true, + }, + { + name: "invalid create-resource, missing name", + spec: &kube.KubernetesSpec{ + CreateResources: []kube.CreateResource{ + { + File: "../file/path.yaml", + }, + }, + }, + expectedErr: true, + }, + { + name: "invalid create-resources, both file and manifest", + spec: &kube.KubernetesSpec{ + CreateResources: []kube.CreateResource{ + { + Name: "test", + File: "../file/path.yaml", + Manifest: ` +apiVersion: v1 +kind: Pod +metadata: + name: test +`, + }, + }, + }, + expectedErr: true, + }, + { + name: "valid wait", + spec: &kube.KubernetesSpec{ + Wait: &kube.Wait{ + Condition: "Ready", + Kind: "namespace/test", + }, + }, + expectedErr: false, + }, + { + name: "invalid wait, both condition and jsonpath specified", + spec: &kube.KubernetesSpec{ + Wait: &kube.Wait{ + Condition: "Ready", + Jsonpath: "test", + Kind: "namespace/test", + }, + }, + expectedErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := kube.CreateKubernetesDomain(context.Background(), tt.spec) + if (err != nil) != tt.expectedErr { + t.Errorf("CreateKubernetesDomain() error = %v, wantErr %v", err, tt.expectedErr) + } + }) + } +} diff --git a/src/pkg/providers/kyverno/types.go b/src/pkg/providers/kyverno/types.go index 5f25663b..f9b7b679 100644 --- a/src/pkg/providers/kyverno/types.go +++ b/src/pkg/providers/kyverno/types.go @@ -2,6 +2,7 @@ package kyverno import ( "context" + "fmt" "github.com/defenseunicorns/lula/src/types" kjson "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" @@ -15,6 +16,22 @@ type KyvernoProvider struct { Spec *KyvernoSpec `json:"spec,omitempty" yaml:"spec,omitempty"` } +func CreateKyvernoProvider(ctx context.Context, spec *KyvernoSpec) (types.Provider, error) { + // Check validity of spec + if spec == nil { + return nil, fmt.Errorf("spec is nil") + } + + if spec.Policy == nil { + return nil, fmt.Errorf("policy is nil") + } + + return KyvernoProvider{ + Context: ctx, + Spec: spec, + }, nil +} + func (k KyvernoProvider) Evaluate(resources types.DomainResources) (types.Result, error) { results, err := GetValidatedAssets(k.Context, k.Spec.Policy, resources, k.Spec.Output) if err != nil { diff --git a/src/pkg/providers/kyverno/types_test.go b/src/pkg/providers/kyverno/types_test.go new file mode 100644 index 00000000..e32fc1f8 --- /dev/null +++ b/src/pkg/providers/kyverno/types_test.go @@ -0,0 +1,71 @@ +package kyverno_test + +import ( + "context" + "testing" + + "github.com/defenseunicorns/lula/src/pkg/providers/kyverno" + kjson "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" +) + +func TestCreateKyvernoProvider(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + spec *kyverno.KyvernoSpec + wantErr bool + }{ + { + name: "valid spec", + spec: &kyverno.KyvernoSpec{ + Policy: &kjson.ValidatingPolicy{ + Spec: kjson.ValidatingPolicySpec{}, + }, + }, + wantErr: false, + }, + { + name: "valid spec with output", + spec: &kyverno.KyvernoSpec{ + Policy: &kjson.ValidatingPolicy{ + Spec: kjson.ValidatingPolicySpec{}, + }, + Output: &kyverno.KyvernoOutput{ + Validation: "validate.result", + Observations: []string{ + "validate.observation", + }, + }, + }, + wantErr: false, + }, + { + name: "nil spec", + spec: nil, + wantErr: true, + }, + { + name: "nil policy", + spec: &kyverno.KyvernoSpec{}, + wantErr: true, + }, + { + name: "empty policy", + spec: &kyverno.KyvernoSpec{ + Policy: &kjson.ValidatingPolicy{}, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := kyverno.CreateKyvernoProvider(context.Background(), tt.spec) + if (err != nil) != tt.wantErr { + t.Errorf("CreateKyvernoProvider() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} diff --git a/src/pkg/providers/opa/opa.go b/src/pkg/providers/opa/opa.go index 85983d2d..f3b82dc7 100644 --- a/src/pkg/providers/opa/opa.go +++ b/src/pkg/providers/opa/opa.go @@ -16,10 +16,7 @@ func GetValidatedAssets(ctx context.Context, regoPolicy string, dataset map[stri var matchResult types.Result if len(dataset) == 0 { - // Not an error but no entries to validate - // TODO: add a warning log - matchResult.Observations = map[string]string{"OPA validation not performed": "No resources to validate"} - return matchResult, nil + return matchResult, fmt.Errorf("opa validation not performed - no resources to validate") } if output == nil { diff --git a/src/pkg/providers/opa/types.go b/src/pkg/providers/opa/types.go index 72f531cd..912cecb3 100644 --- a/src/pkg/providers/opa/types.go +++ b/src/pkg/providers/opa/types.go @@ -2,6 +2,8 @@ package opa import ( "context" + "fmt" + "strings" "github.com/defenseunicorns/lula/src/types" ) @@ -14,6 +16,37 @@ type OpaProvider struct { Spec *OpaSpec `json:"spec,omitempty" yaml:"spec,omitempty"` } +func CreateOpaProvider(ctx context.Context, spec *OpaSpec) (types.Provider, error) { + // Check validity of spec + if spec == nil { + return nil, fmt.Errorf("spec is nil") + } + + if spec.Rego == "" { + return nil, fmt.Errorf("rego policy cannot be empty") + } + + if spec.Output != nil { + if spec.Output.Validation != "" { + if !strings.Contains(spec.Output.Validation, ".") { + return nil, fmt.Errorf("validation field must be a json path") + } + } + if spec.Output.Observations != nil { + for _, observation := range spec.Output.Observations { + if !strings.Contains(observation, ".") { + return nil, fmt.Errorf("observation field must be a json path") + } + } + } + } + + return OpaProvider{ + Context: ctx, + Spec: spec, + }, nil +} + func (o OpaProvider) Evaluate(resources types.DomainResources) (types.Result, error) { results, err := GetValidatedAssets(o.Context, o.Spec.Rego, resources, o.Spec.Output) if err != nil { diff --git a/src/pkg/providers/opa/types_test.go b/src/pkg/providers/opa/types_test.go new file mode 100644 index 00000000..eed2ec56 --- /dev/null +++ b/src/pkg/providers/opa/types_test.go @@ -0,0 +1,81 @@ +package opa_test + +import ( + "context" + "testing" + + "github.com/defenseunicorns/lula/src/pkg/providers/opa" +) + +func TestCreateOpaProvider(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + spec *opa.OpaSpec + wantErr bool + }{ + { + name: "valid spec", + spec: &opa.OpaSpec{ + Rego: "package validate\n\ndefault validate = false", + }, + wantErr: false, + }, + { + name: "valid spec with output", + spec: &opa.OpaSpec{ + Rego: "package validate\n\ndefault validate = false", + Output: &opa.OpaOutput{ + Validation: "validate.result", + Observations: []string{ + "validate.observation", + }, + }, + }, + wantErr: false, + }, + { + name: "nil spec", + spec: nil, + wantErr: true, + }, + { + name: "empty rego", + spec: &opa.OpaSpec{ + Rego: "", + }, + wantErr: true, + }, + { + name: "invalid validation path", + spec: &opa.OpaSpec{ + Rego: "package validate\n\ndefault validate = false", + Output: &opa.OpaOutput{ + Validation: "invalid-path", + }, + }, + wantErr: true, + }, + { + name: "invalid observation path", + spec: &opa.OpaSpec{ + Rego: "package validate\n\ndefault validate = false", + Output: &opa.OpaOutput{ + Observations: []string{"invalid-path"}, + }, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := opa.CreateOpaProvider(context.Background(), tt.spec) + if (err != nil) != tt.wantErr { + t.Errorf("CreateOpaProvider() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} diff --git a/src/test/e2e/pod_validation_test.go b/src/test/e2e/pod_validation_test.go index 396a0e72..8d3bf1f5 100644 --- a/src/test/e2e/pod_validation_test.go +++ b/src/test/e2e/pod_validation_test.go @@ -3,6 +3,7 @@ package test import ( "context" "os" + "strings" "testing" "time" @@ -12,9 +13,11 @@ import ( "github.com/defenseunicorns/go-oscal/src/pkg/versioning" oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2" "github.com/defenseunicorns/lula/src/cmd/validate" + "github.com/defenseunicorns/lula/src/pkg/common" "github.com/defenseunicorns/lula/src/pkg/common/oscal" "github.com/defenseunicorns/lula/src/pkg/message" "github.com/defenseunicorns/lula/src/test/util" + "github.com/defenseunicorns/lula/src/types" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" @@ -82,11 +85,13 @@ func TestPodLabelValidation(t *testing.T) { }). Assess("Validate pod label", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { oscalPath := "./scenarios/pod-label/oscal-component.yaml" - return validatePodLabelFail(ctx, t, config, oscalPath) + validatePodLabelFail(t, oscalPath) + return ctx }). Assess("Validate pod label (Kyverno)", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { oscalPath := "./scenarios/pod-label/oscal-component-kyverno.yaml" - return validatePodLabelFail(ctx, t, config, oscalPath) + validatePodLabelFail(t, oscalPath) + return ctx }). Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { pod := ctx.Value("test-pod-label").(*corev1.Pod) @@ -101,7 +106,7 @@ func TestPodLabelValidation(t *testing.T) { return ctx }).Feature() - featureBadValidation := features.New("Check Graceful Failure - all not-satisfied without error"). + featureBadValidation := features.New("Check Graceful Failure - check all not-satisfied and matching error"). Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { pod, err := util.GetPod("./scenarios/pod-label/pod.pass.yaml") if err != nil { @@ -118,7 +123,64 @@ func TestPodLabelValidation(t *testing.T) { }). Assess("All not-satisfied", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { oscalPath := "./scenarios/pod-label/oscal-component-all-bad.yaml" - return validatePodLabelFail(ctx, t, config, oscalPath) + findings, observations := validatePodLabelFail(t, oscalPath) + observationRemarksMap := generateObservationRemarksMap(*observations) + + for _, f := range *findings { + // relatedobservations should have len = 1 + relatedObs := *f.RelatedObservations + if f.RelatedObservations == nil || len(relatedObs) != 1 { + t.Fatal("RelatedObservations should have len = 1") + } + remarks, found := observationRemarksMap[relatedObs[0].ObservationUuid] + if !found { + t.Fatal("RelatedObservation not found in map") + } + + switch f.Target.TargetId { + case "ID-1": + if !strings.Contains(remarks, common.ErrInvalidDomain.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrInvalidDomain") + } + case "ID-1.1": + if !strings.Contains(remarks, common.ErrInvalidProvider.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrInvalidProvider") + } + case "ID-2": + if !strings.Contains(remarks, common.ErrInvalidSchema.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrInvalidSchema") + } + case "ID-3": + if !strings.Contains(remarks, common.ErrInvalidYaml.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrInvalidYaml") + } + case "ID-3.1": + if !strings.Contains(remarks, common.ErrInvalidYaml.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrInvalidYaml") + } + case "ID-4": + if !strings.Contains(remarks, types.ErrProviderEvaluate.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrProviderEvaluate") + } + case "ID-5": + if !strings.Contains(remarks, types.ErrDomainGetResources.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrDomainGetResources") + } + case "ID-5.1": + if !strings.Contains(remarks, types.ErrDomainGetResources.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrDomainGetResources") + } + case "ID-5.2": + if !strings.Contains(remarks, types.ErrDomainGetResources.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrDomainGetResources") + } + case "ID-6": + if !strings.Contains(remarks, types.ErrExecutionNotAllowed.Error()) { + t.Fatal("ID-1 - Remarks should contain ErrExecutionNotAllowed") + } + } + } + return ctx }). Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { pod := ctx.Value("test-pod-label").(*corev1.Pod) @@ -245,8 +307,10 @@ func validatePodLabelPass(ctx context.Context, t *testing.T, config *envconf.Con return ctx } -func validatePodLabelFail(ctx context.Context, t *testing.T, config *envconf.Config, oscalPath string) context.Context { +func validatePodLabelFail(t *testing.T, oscalPath string) (*[]oscalTypes_1_1_2.Finding, *[]oscalTypes_1_1_2.Observation) { message.NoProgress = true + validate.ConfirmExecution = false + validate.RunNonInteractively = true assessment, err := validate.ValidateOnPath(oscalPath, "") if err != nil { @@ -269,6 +333,20 @@ func validatePodLabelFail(ctx context.Context, t *testing.T, config *envconf.Con t.Fatal("State should be not-satisfied, but got :", state) } } + return result.Findings, result.Observations +} + +func generateObservationRemarksMap(observations []oscalTypes_1_1_2.Observation) map[string]string { + observationMap := make(map[string]string, len(observations)) + + for i := range observations { + observation := &observations[i] + relevantEvidence := strings.Builder{} + for _, re := range *observation.RelevantEvidence { + relevantEvidence.WriteString(re.Remarks) + } + observationMap[observation.UUID] = relevantEvidence.String() + } - return ctx + return observationMap } diff --git a/src/test/e2e/scenarios/pod-label/oscal-component-all-bad.yaml b/src/test/e2e/scenarios/pod-label/oscal-component-all-bad.yaml index cdc7dc52..342ca08f 100644 --- a/src/test/e2e/scenarios/pod-label/oscal-component-all-bad.yaml +++ b/src/test/e2e/scenarios/pod-label/oscal-component-all-bad.yaml @@ -39,7 +39,7 @@ component-definition: links: - href: "#88AB3470-B96B-4D7C-BC36-02BF9563C46C" rel: lula - text: Bad kubernetes-spec + text: Bad kubernetes-spec, bad wait definition -> ErrInvalidDomain - uuid: bfda0b37-26dd-41be-97a9-06efe13a28d9 control-id: ID-1.1 description: >- @@ -47,9 +47,9 @@ component-definition: quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. links: - - href: "#7a7803b5-1da3-4744-af16-d2b81908ade0" + - href: "#01e21994-2cfc-45fb-ac84-d00f2e5912b0" rel: lula - text: Bad api-spec + text: Bad opa-spec, bad output.validation -> ErrInvalidProvider - uuid: 86a0e8d9-0ce0-4304-afe7-4c000001e032 control-id: ID-2 description: >- @@ -57,30 +57,30 @@ component-definition: quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. links: - - href: "#01e21994-2cfc-45fb-ac84-d00f2e5912b0" + - href: "#7a7803b5-1da3-4744-af16-d2b81908ade0" rel: lula - text: Bad opa-spec - - uuid: 99f74603-52d7-4211-9b0f-752a29fd43e7 - control-id: ID-2.1 + text: Bad api-spec, empty requests -> ErrInvalidSchema + - uuid: 80485783-40d7-45ad-bacd-70fb0a1dc03c + control-id: ID-3 description: >- Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. links: - - href: "#7b136728-89c8-4cd6-b3aa-d10679cf4136" + - href: "#b57ed0ba-78f7-4a46-b0b1-d7328ce7347c" rel: lula - text: Bad kyverno-spec - - uuid: 80485783-40d7-45ad-bacd-70fb0a1dc03c - control-id: ID-3 + text: Malformed validation -> ErrInvalidYaml + - uuid: 99f74603-52d7-4211-9b0f-752a29fd43e7 + control-id: ID-3.1 description: >- Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. links: - - href: "#b57ed0ba-78f7-4a46-b0b1-d7328ce7347c" + - href: "#7b136728-89c8-4cd6-b3aa-d10679cf4136" rel: lula - text: Malformed validation - - uuid: 7a37c461-29ef-44a4-8b9c-6e44fbffd436 + text: Bad kyverno-spec, not a valid policy -> ErrInvalidYaml + - uuid: 8b2e57e1-c1ff-44cb-9ced-e052a8af6e7a control-id: ID-4 description: >- Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, @@ -89,7 +89,7 @@ component-definition: links: - href: "#0e8dbc72-9363-466d-9653-dcb7383c2be2" rel: lula - text: Malformed rego + text: Bad opa-spec, invalid rego -> ErrProviderEvaluate - uuid: 3ccbf559-d244-4e0a-aff0-9fe065bb247d control-id: ID-5 description: >- @@ -99,47 +99,37 @@ component-definition: links: - href: "#ea66cb18-d26c-4dd7-8e2b-65999cd542c2" rel: lula - text: No resources by name - - uuid: e308ca47-74c0-4cb7-8fa4-ff40af8ea4bf - control-id: ID-6 - description: >- - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, - quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum - dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. - links: - - href: "#3d170c49-41a7-4677-9b3f-f8c7c0a8ab24" - rel: lula - text: No resources by namespace - - uuid: 36d126f3-f76a-4681-a3d8-d76a0d925522 - control-id: ID-6 + text: No resources by name -> ErrDomainGetResources + - uuid: 7c73052c-0764-4063-a6f1-65db773a8e3e + control-id: ID-5.1 description: >- Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. links: - - href: "#3d170c49-41a7-4677-9b3f-xxxxxxxxxxxx" + - href: "#3d0c3020-7e92-47c1-944e-5667e68fdef8" rel: lula - text: Href does not exist - - uuid: 7c73052c-0764-4063-a6f1-65db773a8e3e - control-id: ID-7 + text: Named resource, namespace-scoped, no namespace -> ErrDomainGetResources + - uuid: 4dac6c27-db71-441a-8704-26b0d313ffa4 + control-id: ID-5.2 description: >- Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. links: - - href: "#3d0c3020-7e92-47c1-944e-5667e68fdef8" + - href: "#97d41576-66c5-448e-a0b1-040020be89a0" rel: lula - text: Named resource, namespace-scoped, no namespace - - uuid: 4dac6c27-db71-441a-8704-26b0d313ffa4 - control-id: ID-8 + text: Named resource, cluster-scoped, with namespace -> ErrDomainGetResources + - uuid: e308ca47-74c0-4cb7-8fa4-ff40af8ea4bf + control-id: ID-6 description: >- Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. links: - - href: "#97d41576-66c5-448e-a0b1-040020be89a0" + - href: "#3d170c49-41a7-4677-9b3f-f8c7c0a8ab24" rel: lula - text: Named resource, cluster-scoped, with namespace + text: Denied execution -> ErrExecutionNotAllowed back-matter: resources: - uuid: 88AB3470-B96B-4D7C-BC36-02BF9563C46C @@ -157,14 +147,7 @@ component-definition: rego: | package validate - import future.keywords.every - - validate { - every pod in input.podsvt { - podLabel := pod.metadata.labels.foo - podLabel == "bar" - } - } + default validate = false - uuid: 7a7803b5-1da3-4744-af16-d2b81908ade0 description: | metadata: @@ -179,14 +162,7 @@ component-definition: rego: | package validate - import future.keywords.every - - validate { - every pod in input.podsvt { - podLabel := pod.metadata.labels.foo - podLabel == "bar" - } - } + default validate = false - uuid: 01e21994-2cfc-45fb-ac84-d00f2e5912b0 description: | metadata: @@ -205,7 +181,12 @@ component-definition: provider: type: opa opa-spec: - no-rego: "nothing" + rego: | + package validate + + default validate = false + output: + validation: some-validation-var - uuid: 7b136728-89c8-4cd6-b3aa-d10679cf4136 description: | metadata: @@ -252,6 +233,7 @@ component-definition: opa-spec: rego: | # No package definition -> invalid rego + default validate = false - uuid: ea66cb18-d26c-4dd7-8e2b-65999cd542c2 description: | @@ -282,13 +264,9 @@ component-definition: domain: type: kubernetes kubernetes-spec: - resources: - - name: podsvt - resource-rule: - group: - version: v1 - resource: pods - namespaces: [no-namespace] + create-resources: + - name: new-pod + file: file:./pod.fail.yaml provider: type: opa opa-spec: diff --git a/src/test/e2e/scenarios/validation-composition/component-definition-bad-href.yaml b/src/test/e2e/scenarios/validation-composition/component-definition-bad-href.yaml new file mode 100644 index 00000000..48c9b52a --- /dev/null +++ b/src/test/e2e/scenarios/validation-composition/component-definition-bad-href.yaml @@ -0,0 +1,51 @@ +component-definition: + components: + - control-implementations: + - description: Validate generic security requirements + implemented-requirements: + - control-id: ID-1 + description: Test control ID-1 - check non-resolvable remote validation is a not-satisfied finding. + links: + # non-resolvable remote validation + - href: https://this-is-a-fake-url.com/validation.yaml + rel: lula + uuid: 42C2FFDC-5F05-44DF-A67F-EEC8660AEFFD + - control-id: ID-2 + description: Test control ID-2 - check non-resolvable local file validation is a not-satisfied finding. + links: + # non-resolvable local file validation + - href: file://./not-a-validation.yaml + rel: lula + uuid: 6c05b0d4-6f08-4c09-8ff3-7f3c06a236e1 + - control-id: ID-3 + description: Test control ID-3 - check bad checksum file validation is a not-satisfied finding. + links: + # single validation w/ bad checksum + - href: file://./validation.opa.yaml@1234abcd + rel: lula + uuid: 82ca233b-157f-4de4-bf79-2c0cefe43335 + source: https://raw.githubusercontent.com/usnistgov/oscal-content/master/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json + uuid: 68e3f5af-6823-4e7e-8cdf-d16916d3ac88 + description: | + Lula - the Compliance Validator + purpose: Validate compliance controls + responsible-roles: + - party-uuids: + - C18F4A9F-A402-415B-8D13-B51739D689FF + role-id: provider + title: lula + type: software + uuid: A9D5204C-7E5B-4C43-BD49-34DF759B9F04 + metadata: + last-modified: 2024-04-03T09:56:20.719564-07:00 + oscal-version: 1.1.2 + parties: + - links: + - href: https://github.com/defenseunicorns/lula + rel: website + name: Lula Development + type: organization + uuid: C18F4A9F-A402-415B-8D13-B51739D689FF + title: Lula Demo + version: "20220913" + uuid: E6A291A4-2BC8-43A0-B4B2-FD67CAAE1F8F diff --git a/src/test/e2e/validation_composition_test.go b/src/test/e2e/validation_composition_test.go index 124f0d64..e6ff1779 100644 --- a/src/test/e2e/validation_composition_test.go +++ b/src/test/e2e/validation_composition_test.go @@ -46,78 +46,11 @@ func TestValidationComposition(t *testing.T) { }). Assess("Validate local composition file", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { oscalPath := "./scenarios/validation-composition/component-definition.yaml" - compDefBytes, err := os.ReadFile(oscalPath) - if err != nil { - t.Error(err) - } - - assessment, err := validate.ValidateOnPath(oscalPath, "") - if err != nil { - t.Fatal(err) - } - - if len(assessment.Results) == 0 { - t.Fatal("Expected greater than zero results") - } - - result := assessment.Results[0] - - if result.Findings == nil { - t.Fatal("Expected findings to be not nil") - } - - for _, finding := range *result.Findings { - state := finding.Target.Status.State - if state != "satisfied" { - t.Fatal("State should be satisfied, but got :", state) - } - } - - var oscalModel oscalTypes_1_1_2.OscalCompleteSchema - err = yaml.Unmarshal(compDefBytes, &oscalModel) - if err != nil { - t.Error(err) - } - reset, err := common.SetCwdToFileDir(oscalPath) - if err != nil { - t.Fatalf("Error setting cwd to file dir: %v", err) - } - defer reset() - - compDef := oscalModel.ComponentDefinition - - err = composition.ComposeComponentValidations(compDef) - if err != nil { - t.Error(err) - } - - components := *compDef.Components - - // Create a validation store from the back-matter if it exists - validationStore := validationstore.NewValidationStoreFromBackMatter(*compDef.BackMatter) - - findingMap, observations, err := validate.ValidateOnControlImplementations(components[0].ControlImplementations, validationStore, "") - if err != nil { - t.Fatalf("Error with validateOnControlImplementations: %v", err) - } - - // For fun - create a result - composeResult, err := oscal.CreateResult(findingMap, observations) - if err != nil { - t.Fatal(err) - } - - // Compare results - status, _, err := oscal.EvaluateResults(&result, &composeResult) - if err != nil { - t.Fatal(err) - } - - if !status { - t.Fatal("Expected Successful evaluate") - } - - return ctx + return validateComposition(ctx, t, oscalPath, "satisfied") + }). + Assess("Validate local composition file with bad links", func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { + oscalPath := "./scenarios/validation-composition/component-definition-bad-href.yaml" + return validateComposition(ctx, t, oscalPath, "not-satisfied") }). Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { @@ -131,3 +64,78 @@ func TestValidationComposition(t *testing.T) { testEnv.Test(t, featureValidationComposition) } + +func validateComposition(ctx context.Context, t *testing.T, oscalPath, expectedFindingState string) context.Context { + compDefBytes, err := os.ReadFile(oscalPath) + if err != nil { + t.Error(err) + } + + assessment, err := validate.ValidateOnPath(oscalPath, "") + if err != nil { + t.Fatal(err) + } + + if len(assessment.Results) == 0 { + t.Fatal("Expected greater than zero results") + } + + result := assessment.Results[0] + + if result.Findings == nil { + t.Fatal("Expected findings to be not nil") + } + + for _, finding := range *result.Findings { + state := finding.Target.Status.State + if state != expectedFindingState { + t.Fatalf("State should be %s, but got: %s", expectedFindingState, state) + } + } + + var oscalModel oscalTypes_1_1_2.OscalCompleteSchema + err = yaml.Unmarshal(compDefBytes, &oscalModel) + if err != nil { + t.Error(err) + } + reset, err := common.SetCwdToFileDir(oscalPath) + if err != nil { + t.Fatalf("Error setting cwd to file dir: %v", err) + } + defer reset() + + compDef := oscalModel.ComponentDefinition + + err = composition.ComposeComponentValidations(compDef) + if err != nil { + t.Error(err) + } + + components := *compDef.Components + + // Create a validation store from the back-matter if it exists + validationStore := validationstore.NewValidationStoreFromBackMatter(*compDef.BackMatter) + + findingMap, observations, err := validate.ValidateOnControlImplementations(components[0].ControlImplementations, validationStore, "") + if err != nil { + t.Fatalf("Error with validateOnControlImplementations: %v", err) + } + + // For fun - create a result + composeResult, err := oscal.CreateResult(findingMap, observations) + if err != nil { + t.Fatal(err) + } + + // Compare results + status, _, err := oscal.EvaluateResults(&result, &composeResult) + if err != nil { + t.Fatal(err) + } + + if !status { + t.Fatal("Expected Successful evaluate") + } + + return ctx +} diff --git a/src/test/unit/common/composition/component-definition-all-local-bad-href.yaml b/src/test/unit/common/composition/component-definition-all-local-bad-href.yaml new file mode 100644 index 00000000..89de19ee --- /dev/null +++ b/src/test/unit/common/composition/component-definition-all-local-bad-href.yaml @@ -0,0 +1,68 @@ +component-definition: + uuid: E6A291A4-2BC8-43A0-B4B2-FD67CAAE1F8F + metadata: + title: Lula Demo + last-modified: "2022-09-13T12:00:00Z" + version: "20220913" + oscal-version: 1.1.2 # This version should remain one version behind latest version for `lula dev upgrade` demo + parties: + # Should be consistent across all of the packages, but where is ground truth? + - uuid: C18F4A9F-A402-415B-8D13-B51739D689FF + type: organization + name: Lula Development + links: + - href: https://github.com/defenseunicorns/lula + rel: website + components: + - uuid: A9D5204C-7E5B-4C43-BD49-34DF759B9F04 + type: software + title: lula + description: | + Lula - the Compliance Validator + purpose: Validate compliance controls + responsible-roles: + - role-id: provider + party-uuids: + - C18F4A9F-A402-415B-8D13-B51739D689FF # matches parties entry for Defense Unicorns + control-implementations: + - uuid: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A + source: https://raw.githubusercontent.com/usnistgov/oscal-content/master/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json + description: Validate generic security requirements + implemented-requirements: + - uuid: 42C2FFDC-5F05-44DF-A67F-EEC8660AEFFD + control-id: ID-1 + description: >- + This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + links: + - href: "#f3c06345-09de-4d00-8a1d-3d66ea57c1fc" + rel: lula + back-matter: + resources: + - uuid: a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + rlinks: + - href: lula.dev + description: >- + domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podsvt + resource-rule: + group: + version: v1 + resource: pods + namespaces: [validation-test] + provider: + type: opa + opa-spec: + rego: | + package validate + + import future.keywords.every + + validate { + every pod in input.podsvt { + podLabel := pod.metadata.labels.foo + podLabel == "bar" + } + } diff --git a/src/test/unit/common/valid-api-spec.yaml b/src/test/unit/common/valid-api-spec.yaml deleted file mode 100644 index 4f0b954c..00000000 --- a/src/test/unit/common/valid-api-spec.yaml +++ /dev/null @@ -1,3 +0,0 @@ -requests: -- name: local - url: "http://localhost" \ No newline at end of file diff --git a/src/test/unit/common/valid-kubernetes-spec.yaml b/src/test/unit/common/valid-kubernetes-spec.yaml deleted file mode 100644 index 0ab89a92..00000000 --- a/src/test/unit/common/valid-kubernetes-spec.yaml +++ /dev/null @@ -1,16 +0,0 @@ -resources: -- name: podsvt - resource-rule: - version: v1 - resource: pods - namespaces: [validation-test] -- name: yamlcm - resource-rule: - name: configmap-yaml - version: v1 - resource: configmaps - namespaces: [validation-test] - field: - jsonpath: .data.app-config.yaml - type: yaml - \ No newline at end of file diff --git a/src/test/unit/common/valid-kyverno-spec.yaml b/src/test/unit/common/valid-kyverno-spec.yaml deleted file mode 100644 index 2d764fa5..00000000 --- a/src/test/unit/common/valid-kyverno-spec.yaml +++ /dev/null @@ -1,15 +0,0 @@ -policy: - apiVersion: json.kyverno.io/v1alpha1 - kind: ValidatingPolicy - metadata: - name: labels - spec: - rules: - - name: foo-label-exists - assert: - all: - - check: - ~.podsvt: - metadata: - labels: - foo: bar \ No newline at end of file diff --git a/src/test/unit/common/valid-opa-spec.yaml b/src/test/unit/common/valid-opa-spec.yaml deleted file mode 100644 index 1f365ee2..00000000 --- a/src/test/unit/common/valid-opa-spec.yaml +++ /dev/null @@ -1,16 +0,0 @@ -rego: | - package validate - - import future.keywords.every - - result { - every pod in input.podsvt { - podLabel := pod.metadata.labels.foo - podLabel == "bar" - } - } - test := "hello world" -output: - validation: validate.result - observations: - - validate.test \ No newline at end of file diff --git a/src/types/types.go b/src/types/types.go index 171da27f..64cc2abe 100644 --- a/src/types/types.go +++ b/src/types/types.go @@ -7,6 +7,13 @@ import ( "github.com/defenseunicorns/lula/src/pkg/message" ) +// Define base errors for validations +var ( + ErrExecutionNotAllowed = errors.New("execution not allowed") + ErrDomainGetResources = errors.New("domain GetResources error") + ErrProviderEvaluate = errors.New("provider Evaluate error") +) + type LulaValidationType string const ( @@ -134,10 +141,10 @@ func (val *LulaValidation) Validate(opts ...LulaValidationOption) error { if config.isInteractive { // Run confirmation user prompt if confirm := message.PromptForConfirmation(config.spinner); !confirm { - return errors.New("execution not allowed") + return fmt.Errorf("%w: requested execution denied", ErrExecutionNotAllowed) } } else { - return errors.New("execution not allowed") + return fmt.Errorf("%w: non-interactive execution not allowed", ErrExecutionNotAllowed) } } } @@ -148,7 +155,7 @@ func (val *LulaValidation) Validate(opts ...LulaValidationOption) error { } else { resources, err = (*val.Domain).GetResources() if err != nil { - return fmt.Errorf("domain GetResources error: %v", err) + return fmt.Errorf("%w: %v", ErrDomainGetResources, err) } if config.onlyResources { return nil @@ -158,7 +165,7 @@ func (val *LulaValidation) Validate(opts ...LulaValidationOption) error { // Perform the evaluation using the provider result, err = (*val.Provider).Evaluate(resources) if err != nil { - return fmt.Errorf("provider Evaluate error: %v", err) + return fmt.Errorf("%w: %v", ErrProviderEvaluate, err) } } return nil