a much better e2e test with templating

defenseunicorns · Oct 29, 2024 · 7d21bf5 · 7d21bf5
1 parent 2b1875c
commit 7d21bf5
Show file tree

Hide file tree

Showing 5 changed files with 313 additions and 5 deletions.
diff --git a/src/pkg/common/types.go b/src/pkg/common/types.go
@@ -10,6 +10,8 @@ import (
 	"github.com/defenseunicorns/go-oscal/src/pkg/uuid"
 	oscalValidation "github.com/defenseunicorns/go-oscal/src/pkg/validation"
 	oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2"
+	"sigs.k8s.io/yaml"
+
 	"github.com/defenseunicorns/lula/src/config"
 	"github.com/defenseunicorns/lula/src/pkg/common/schemas"
 	"github.com/defenseunicorns/lula/src/pkg/domains/api"
@@ -18,7 +20,6 @@ import (
 	"github.com/defenseunicorns/lula/src/pkg/providers/kyverno"
 	"github.com/defenseunicorns/lula/src/pkg/providers/opa"
 	"github.com/defenseunicorns/lula/src/types"
-	"sigs.k8s.io/yaml"
 )
 
 // Define base errors for validations
@@ -153,7 +154,8 @@ func (validation *Validation) ToLulaValidation(uuid string) (lulaValidation type
 	domain, err := GetDomain(validation.Domain)
 	if domain == nil {
 		return lulaValidation, fmt.Errorf("%w: %s", ErrInvalidDomain, validation.Domain.Type)
-	} else if err != nil {
+	}
+	if err != nil {
 		return lulaValidation, fmt.Errorf("%w: %v", ErrInvalidDomain, err)
 	}
 	lulaValidation.Domain = &domain

diff --git a/src/pkg/domains/api/api.go b/src/pkg/domains/api/api.go
@@ -30,9 +30,8 @@ func (a ApiDomain) makeRequests(ctx context.Context) (types.DomainResources, err
 		defaultClient := clientFromOpts(defaultOpts)
 
 		for _, request := range a.Spec.Requests {
-			var responseType map[string]interface{}
+			var responseType interface{}
 			var err error
-
 			if request.Options == nil {
 				responseType, err = doHTTPReq(defaultClient, *request.reqURL, defaultOpts.Headers, request.reqParameters, responseType)
 			} else {
@@ -42,7 +41,6 @@ func (a ApiDomain) makeRequests(ctx context.Context) (types.DomainResources, err
 			if err != nil {
 				return nil, err
 			}
-
 			collection[request.Name] = responseType
 		}
 		return collection, nil

diff --git a/src/test/e2e/api_validation_test.go b/src/test/e2e/api_validation_test.go
@@ -2,15 +2,21 @@ package test
 
 import (
 	"context"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/e2e-framework/klient/wait"
 	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/pkg/features"
 
+	"github.com/defenseunicorns/lula/src/cmd/dev"
+	"github.com/defenseunicorns/lula/src/internal/template"
+	"github.com/defenseunicorns/lula/src/pkg/common/composition"
 	"github.com/defenseunicorns/lula/src/pkg/common/validation"
 	"github.com/defenseunicorns/lula/src/pkg/message"
 	"github.com/defenseunicorns/lula/src/test/util"
@@ -204,3 +210,42 @@ func TestApiValidation(t *testing.T) {
 
 	testEnv.Test(t, featureTrueValidation, featureFalseValidation)
 }
+
+func TestApiValidation_templated(t *testing.T) {
+	message.NoProgress = true
+	dev.RunInteractively = false
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"pass": true}`))
+	}))
+	defer svr.Close()
+
+	tmpl := "scenarios/api-validations/component-definition.yaml.tmpl"
+	composer, err := composition.New(
+		composition.WithModelFromLocalPath(tmpl),
+		composition.WithRenderSettings("all", true),
+		composition.WithTemplateRenderer("all", nil, []template.VariableConfig{
+			{
+				Key:     "reqUrl",
+				Default: svr.URL,
+			},
+		}, []string{}),
+	)
+	require.NoError(t, err)
+
+	validator, err := validation.New(validation.WithComposition(composer, tmpl))
+	require.NoError(t, err)
+
+	assessment, err := validator.ValidateOnPath(context.Background(), tmpl, "")
+	require.NoError(t, err)
+	require.GreaterOrEqual(t, len(assessment.Results), 1)
+
+	result := assessment.Results[0]
+	require.NotNil(t, result.Findings)
+	for _, finding := range *result.Findings {
+		state := finding.Target.Status.State
+		if state != "satisfied" {
+			t.Fatal("State should be satisfied, but got :", state)
+		}
+	}
+}
diff --git a/src/test/e2e/scenarios/api-validations/component-definition.yaml.tmpl b/src/test/e2e/scenarios/api-validations/component-definition.yaml.tmpl
@@ -0,0 +1,67 @@
+component-definition:
+  uuid: E6A291A4-2BC8-43A0-B4B2-FD67CAAE1F8F
+  metadata:
+    title: OSCAL Demo Tool
+    last-modified: "2022-09-13T12:00:00Z"
+    version: "20220913"
+    oscal-version: 1.1.1
+    parties:
+      - uuid: C18F4A9F-A402-415B-8D13-B51739D689FF
+        type: organization
+        name: Defense Unicorns
+        links:
+          - href: https://github.com/defenseunicorns/lula
+            rel: website
+  components:
+    - uuid: A9D5204C-7E5B-4C43-BD49-34DF759B9F04
+      type: software
+      title: lula
+      description: |
+        Defense Unicorns lula
+      purpose: Validate compliance controls
+      responsible-roles:
+        - role-id: provider
+          party-uuids:
+            - C18F4A9F-A402-415B-8D13-B51739D689FF
+      control-implementations:
+        - uuid: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A
+          source: https://github.com/defenseunicorns/lula
+          description: Validate generic security requirements
+          implemented-requirements:
+            - uuid: 2851DD23-03D7-4245-B939-25F11F635359
+              control-id: ID-1
+              description: >-
+                NOT Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 
+                quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum 
+                dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+              links:
+                - href: "#C30E849E-C262-42DF-8C84-EA1B62A6AD90"
+                  rel: lula
+  back-matter:
+    resources:
+      - uuid: C30E849E-C262-42DF-8C84-EA1B62A6AD90
+        description: >-
+          metadata:
+            name: test pass
+            uuid: 88AB3470-B96B-4D7C-BC36-02BF9563C46C
+          domain:
+            type: api
+            api-spec:
+              options:
+                timeout: 15s
+                headers:
+                  x-special-header: "lula"
+              requests:
+              - name: healthcheck
+                url: {{ .var.reqUrl }}
+                parameters:
+                  key: value
+          provider: 
+            type: opa
+            opa-spec:
+              rego: |
+                package validate
+
+                validate {
+                  input.healthcheck.pass == true
+                }
diff --git a/src/test/e2e/scenarios/api-validations/out.yaml b/src/test/e2e/scenarios/api-validations/out.yaml
@@ -0,0 +1,196 @@
+assessment-results:
+  import-ap:
+    href: ""
+  metadata:
+    last-modified: 2024-10-29T15:47:14.786284-04:00
+    oscal-version: 1.1.2
+    published: 2024-10-29T15:44:33.880441-04:00
+    remarks: Assessment Results generated from Lula
+    title: '[System Name] Security Assessment Results (SAR)'
+    version: 0.0.1
+  results:
+    - description: Assessment results for performing Validations with Lula version unset
+      findings:
+        - description: |
+            Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Implemented Requirement: 2851DD23-03D7-4245-B939-25F11F635359
+            NOT Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,  quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum  dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+          related-observations:
+            - observation-uuid: 95049f8b-2fec-475b-9c66-54be2fb33723
+          target:
+            status:
+              state: not-satisfied
+            target-id: ID-1
+            type: objective-id
+          title: 'Validation Result - Control: ID-1'
+          uuid: e1beef74-6fc7-487d-adf9-4cf5acd0aee6
+      observations:
+        - collected: 2024-10-29T15:47:14.770886-04:00
+          description: |
+            [TEST]: C30E849E-C262-42DF-8C84-EA1B62A6AD90 - test pass
+          methods:
+            - TEST
+          props:
+            - name: validation
+              ns: https://docs.lula.dev/oscal/ns
+              value: '#C30E849E-C262-42DF-8C84-EA1B62A6AD90'
+          relevant-evidence:
+            - description: |
+                Result: not-satisfied
+          uuid: 95049f8b-2fec-475b-9c66-54be2fb33723
+      props:
+        - name: threshold
+          ns: https://docs.lula.dev/oscal/ns
+          value: "false"
+        - name: target
+          ns: https://docs.lula.dev/oscal/ns
+          value: https://github.com/defenseunicorns/lula
+      reviewed-controls:
+        control-selections:
+          - description: Controls Assessed by Lula
+            include-controls:
+              - control-id: ID-1
+        description: Controls validated
+        remarks: Validation performed may indicate full or partial satisfaction
+      start: 2024-10-29T15:47:14.771041-04:00
+      title: Lula Validation Result
+      uuid: 2a73b7ae-1310-49cf-9fc4-3a558de2dce3
+    - description: Assessment results for performing Validations with Lula version unset
+      findings:
+        - description: |
+            Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Implemented Requirement: 2851DD23-03D7-4245-B939-25F11F635359
+            NOT Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,  quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum  dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+          related-observations:
+            - observation-uuid: 140a6258-f47d-4f9f-9250-4e05c93ecbaf
+          target:
+            status:
+              state: not-satisfied
+            target-id: ID-1
+            type: objective-id
+          title: 'Validation Result - Control: ID-1'
+          uuid: d3924bf7-3eae-4212-9227-036ef7ed0143
+      observations:
+        - collected: 2024-10-29T15:45:40.595569-04:00
+          description: |
+            [TEST]: C30E849E-C262-42DF-8C84-EA1B62A6AD90 - test pass
+          methods:
+            - TEST
+          props:
+            - name: validation
+              ns: https://docs.lula.dev/oscal/ns
+              value: '#C30E849E-C262-42DF-8C84-EA1B62A6AD90'
+          relevant-evidence:
+            - description: |
+                Result: not-satisfied
+          uuid: 140a6258-f47d-4f9f-9250-4e05c93ecbaf
+      props:
+        - name: threshold
+          ns: https://docs.lula.dev/oscal/ns
+          value: "false"
+        - name: target
+          ns: https://docs.lula.dev/oscal/ns
+          value: https://github.com/defenseunicorns/lula
+      reviewed-controls:
+        control-selections:
+          - description: Controls Assessed by Lula
+            include-controls:
+              - control-id: ID-1
+        description: Controls validated
+        remarks: Validation performed may indicate full or partial satisfaction
+      start: 2024-10-29T15:45:40.595842-04:00
+      title: Lula Validation Result
+      uuid: 08a6a242-a30d-477b-a9ea-3d5c8fc9c669
+    - description: Assessment results for performing Validations with Lula version unset
+      findings:
+        - description: |
+            Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Implemented Requirement: 2851DD23-03D7-4245-B939-25F11F635359
+            NOT Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,  quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum  dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+          related-observations:
+            - observation-uuid: c81557b4-b5e5-4bc8-a0f3-a572cdc8333a
+          target:
+            status:
+              state: not-satisfied
+            target-id: ID-1
+            type: objective-id
+          title: 'Validation Result - Control: ID-1'
+          uuid: 4ca46f3a-1c9e-4d33-a9df-063d32adc8af
+      observations:
+        - collected: 2024-10-29T15:45:12.84235-04:00
+          description: |
+            [TEST]: C30E849E-C262-42DF-8C84-EA1B62A6AD90 - test pass
+          methods:
+            - TEST
+          props:
+            - name: validation
+              ns: https://docs.lula.dev/oscal/ns
+              value: '#C30E849E-C262-42DF-8C84-EA1B62A6AD90'
+          relevant-evidence:
+            - description: |
+                Result: not-satisfied
+              remarks: |
+                Error running validation: provider Evaluate error: opa validation not performed - no resources to validate
+          uuid: c81557b4-b5e5-4bc8-a0f3-a572cdc8333a
+      props:
+        - name: threshold
+          ns: https://docs.lula.dev/oscal/ns
+          value: "false"
+        - name: target
+          ns: https://docs.lula.dev/oscal/ns
+          value: https://github.com/defenseunicorns/lula
+      reviewed-controls:
+        control-selections:
+          - description: Controls Assessed by Lula
+            include-controls:
+              - control-id: ID-1
+        description: Controls validated
+        remarks: Validation performed may indicate full or partial satisfaction
+      start: 2024-10-29T15:45:12.842481-04:00
+      title: Lula Validation Result
+      uuid: f6596acf-4307-436e-b3ef-fc82b2346c2f
+    - description: Assessment results for performing Validations with Lula version unset
+      findings:
+        - description: |
+            Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Implemented Requirement: 2851DD23-03D7-4245-B939-25F11F635359
+            NOT Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,  quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum  dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+          related-observations:
+            - observation-uuid: da9f7464-ec9a-4e3c-8d52-19e399ffae85
+          target:
+            status:
+              state: not-satisfied
+            target-id: ID-1
+            type: objective-id
+          title: 'Validation Result - Control: ID-1'
+          uuid: 7a52b085-9322-417e-af20-1dc2fad5727d
+      observations:
+        - collected: 2024-10-29T15:44:33.880056-04:00
+          description: |
+            [TEST]: C30E849E-C262-42DF-8C84-EA1B62A6AD90 - test pass
+          methods:
+            - TEST
+          props:
+            - name: validation
+              ns: https://docs.lula.dev/oscal/ns
+              value: '#C30E849E-C262-42DF-8C84-EA1B62A6AD90'
+          relevant-evidence:
+            - description: |
+                Result: not-satisfied
+              remarks: |
+                Error running validation: provider Evaluate error: opa validation not performed - no resources to validate
+          uuid: da9f7464-ec9a-4e3c-8d52-19e399ffae85
+      props:
+        - name: threshold
+          ns: https://docs.lula.dev/oscal/ns
+          value: "false"
+        - name: target
+          ns: https://docs.lula.dev/oscal/ns
+          value: https://github.com/defenseunicorns/lula
+      reviewed-controls:
+        control-selections:
+          - description: Controls Assessed by Lula
+            include-controls:
+              - control-id: ID-1
+        description: Controls validated
+        remarks: Validation performed may indicate full or partial satisfaction
+      start: 2024-10-29T15:44:33.880439-04:00
+      title: Lula Validation Result
+      uuid: 5bc18561-4830-40df-ad0e-9f34c1f6ee03
+  uuid: 987e34a6-8be7-4322-be93-7b366043d0eb