fix(evaluate): separate write logic from core evaluation logic

src/cmd/evaluate/evaluate.go

@@ -35,10 +35,20 @@ var evaluateCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 
 		// Access the files and evaluate them
-		err := EvaluateAssessmentResults(opts.files)
+		assessmentMap, err := EvaluateAssessmentResults(opts.files)
 		if err != nil {
 			message.Fatal(err, err.Error())
 		}
+
+		// Props are updated - now write back to all files
+		// if we create the model and write it - the merge will need to de-duplicate instead of merge results
+		for filePath, assessment := range assessmentMap {
+			model := oscalTypes_1_1_2.OscalCompleteSchema{
+				AssessmentResults: assessment,
+			}
+
+			oscal.WriteOscalModel(filePath, &model)
+		}
 	},
 }
 
@@ -49,30 +59,30 @@ func EvaluateCommand() *cobra.Command {
 	return evaluateCmd
 }
 
-func EvaluateAssessmentResults(fileArray []string) error {
+func EvaluateAssessmentResults(fileArray []string) (map[string]*oscalTypes_1_1_2.AssessmentResults, error) {
 	var status bool
 	var findings map[string][]oscalTypes_1_1_2.Finding
 	var threshold, latest *oscalTypes_1_1_2.Result
 
 	if len(fileArray) == 0 {
-		return fmt.Errorf("no files provided for evaluation")
+		return nil, fmt.Errorf("no files provided for evaluation")
 	}
 
 	// Potentially write changes back to multiple files requires some storage
 	resultMap := make(map[string]*oscalTypes_1_1_2.AssessmentResults)
 	for _, fileString := range fileArray {
 		err := files.IsJsonOrYaml(fileString)
 		if err != nil {
-			return fmt.Errorf("invalid file extension: %s, requires .json or .yaml", fileString)
+			return nil, fmt.Errorf("invalid file extension: %s, requires .json or .yaml", fileString)
 		}
 
 		data, err := common.ReadFileToBytes(fileString)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		assessment, err := oscal.NewAssessmentResults(data)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		resultMap[fileString] = assessment
 	}
@@ -82,13 +92,13 @@ func EvaluateAssessmentResults(fileArray []string) error {
 
 	thresholds, sortedResults, err := findAndSortResults(resultMap)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	if len(sortedResults) <= 1 {
 		// Should this implicitly pass? If so then a workflow can operate on the assumption that it will pass from 0 -> N results
 		message.Infof("%v result object identified - unable to evaluate", len(sortedResults))
-		return nil
+		return nil, nil
 	}
 
 	if len(thresholds) == 0 {
@@ -98,7 +108,7 @@ func EvaluateAssessmentResults(fileArray []string) error {
 
 		status, findings, err = EvaluateResults(threshold, latest)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	} else {
 		// Constraint - Always evaluate the latest threshold against the latest result
@@ -107,11 +117,11 @@ func EvaluateAssessmentResults(fileArray []string) error {
 
 		if threshold.UUID == latest.UUID {
 			// They are the same - return error
-			return fmt.Errorf("unable to evaluate - threshold and latest result are the same result - nothing to compare")
+			return nil, fmt.Errorf("unable to evaluate - threshold and latest result are the same result - nothing to compare")
 		}
 		status, findings, err = EvaluateResults(threshold, latest)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
 
@@ -122,21 +132,13 @@ func EvaluateAssessmentResults(fileArray []string) error {
 				message.Infof("%s", finding.Target.TargetId)
 			}
 
-			message.Info("New threshold identified - threshold will be updated to latest result")
-
-			updateProp("threshold", "false", threshold.Props)
-			updateProp("threshold", "true", latest.Props)
-
-			// Props are updated - now write back to all files
-			// if we create the model and write it - the merge will need to de-duplicate instead of merge results
-			for filePath, assessment := range resultMap {
-				model := oscalTypes_1_1_2.OscalCompleteSchema{
-					AssessmentResults: assessment,
-				}
+			message.Infof("New threshold identified - threshold will be updated to result %s", latest.UUID)
 
-				oscal.WriteOscalModel(filePath, &model)
+			// In the event we still have multiple thresholds - let's clean them up
+			for _, result := range thresholds {
+				updateProp("threshold", "false", result.Props)
 			}
-
+			updateProp("threshold", "true", latest.Props)
 		}
 
 		if len(findings["new-failing-findings"]) > 0 {
@@ -146,13 +148,13 @@ func EvaluateAssessmentResults(fileArray []string) error {
 			}
 		}
 
-		return nil
+		return resultMap, nil
 	} else {
 		message.Warn("Evaluation Failed against the following findings:")
 		for _, finding := range findings["no-longer-satisfied"] {
 			message.Warnf("%s", finding.Target.TargetId)
 		}
-		return fmt.Errorf("failed to meet established threshold")
+		return nil, fmt.Errorf("failed to meet established threshold")
 	}
 }
 

src/cmd/evaluate/evaluate_test.go

@@ -7,6 +7,41 @@ import (
 	"github.com/defenseunicorns/lula/src/pkg/message"
 )
 
+var (
+	validInputFile   = "../../test/unit/common/oscal/valid-assessment-result.yaml"
+	invalidInputFile = "../../test/unit/common/oscal/invalid-assessment-result.yaml"
+)
+
+func TestEvaluateAssessmentResults(t *testing.T) {
+	t.Parallel()
+
+	// TODO: write logic to separate file read from core evaluation logic
+	// TODO: move the core logic to library package
+	// TODO: write a success test to receive assessments/results that we can verify intended prop change
+	// t.Run("handles valid assessment result", func(t *testing.T) {
+	// 	assessmentMap, err := EvaluateAssessmentResults([]string{validInputFile})
+	// 	if err != nil {
+	// 		t.Fatal("unexpected error for valid assessment result")
+	// 	}
+
+	// })
+
+	t.Run("handles invalid path to assessment result file", func(t *testing.T) {
+		_, err := EvaluateAssessmentResults([]string{"./invalid-path.yaml"})
+		if err == nil {
+			t.Fatal("expected error for invalid path")
+		}
+	})
+
+	t.Run("handles invalid assessment result without any results", func(t *testing.T) {
+		_, err := EvaluateAssessmentResults([]string{invalidInputFile})
+		if err == nil {
+			t.Fatal("expected error for invalid assessment result without results")
+		}
+	})
+
+}
+
 // Given two results - evaluate for passing
 func TestEvaluateResultsPassing(t *testing.T) {
 	message.NoProgress = true

src/test/unit/common/oscal/invalid-assessment-result.yaml

@@ -0,0 +1,11 @@
+assessment-results:
+  import-ap:
+    href: ""
+  metadata:
+    last-modified: 2024-06-04T02:34:02.653177582Z
+    oscal-version: 1.1.2
+    published: 2024-06-04T02:34:02.653177582Z
+    remarks: Assessment Results generated from Lula
+    title: '[System Name] Security Assessment Results (SAR)'
+    version: 0.0.1
+  uuid: 19541f5a-7738-4d86-b53f-34f3dc7a1586

src/test/unit/common/oscal/valid-assessment-result.yaml

@@ -0,0 +1,181 @@
+assessment-results:
+  import-ap:
+    href: ""
+  metadata:
+    last-modified: 2024-06-04T03:34:46.521030308Z
+    oscal-version: 1.1.2
+    published: 2024-06-04T02:46:29.02587111Z
+    remarks: Assessment Results generated from Lula
+    title: '[System Name] Security Assessment Results (SAR)'
+    version: 0.0.1
+  results:
+    - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43
+      findings:
+        - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance.
+          related-observations:
+            - observation-uuid: bcb5c77a-981d-4698-acd4-84c3e8a0b3e7
+          target:
+            status:
+              state: satisfied
+            target-id: ID-1
+            type: objective-id
+          title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control:  ID-1'
+          uuid: 41018849-9825-4760-82ad-6d74362e44d5
+        - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance.
+          related-observations:
+            - observation-uuid: 38d1abe2-7c30-4eaa-b792-46a0978bf0a0
+          target:
+            status:
+              state: satisfied
+            target-id: ID-2
+            type: objective-id
+          title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: 60345fd5-f0b8-4097-b9a2-b4f795edcab3 / Control:  ID-2'
+          uuid: 9fd5633e-2509-41bc-b66c-16d30e7dc29e
+      observations:
+        - collected: 2024-06-04T03:34:46.435905414Z
+          description: |+
+            [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9
+
+          methods:
+            - TEST
+          relevant-evidence:
+            - description: |
+                Result: satisfied
+          uuid: bcb5c77a-981d-4698-acd4-84c3e8a0b3e7
+        - collected: 2024-06-04T03:34:46.441327814Z
+          description: |+
+            [TEST]: ID-2 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9
+
+          methods:
+            - TEST
+          relevant-evidence:
+            - description: |
+                Result: satisfied
+          uuid: 38d1abe2-7c30-4eaa-b792-46a0978bf0a0
+      props:
+        - name: threshold
+          ns: https://docs.lula.dev/ns
+          value: "true"
+      reviewed-controls:
+        control-selections:
+          - description: Controls Assessed by Lula
+            include-controls:
+              - control-id: ID-1
+              - control-id: ID-2
+        description: Controls validated
+        remarks: Validation performed may indicate full or partial satisfaction
+      start: 2024-06-04T03:34:46.443161469Z
+      title: Lula Validation Result
+      uuid: b5fb7bd4-c91e-4cd3-8020-ed04aa8d74b5
+    - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43
+      findings:
+        - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance.
+          related-observations:
+            - observation-uuid: 42a8b71d-7d08-42cc-beb4-c1c384c119ef
+          target:
+            status:
+              state: satisfied
+            target-id: ID-1
+            type: objective-id
+          title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control:  ID-1'
+          uuid: 50c818bd-e6be-4870-8766-8a22d18f1af1
+      observations:
+        - collected: 2024-06-04T03:33:33.977370866Z
+          description: |+
+            [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9
+
+          methods:
+            - TEST
+          relevant-evidence:
+            - description: |
+                Result: satisfied
+          uuid: 42a8b71d-7d08-42cc-beb4-c1c384c119ef
+      props:
+        - name: threshold
+          ns: https://docs.lula.dev/ns
+          value: "false"
+      reviewed-controls:
+        control-selections:
+          - description: Controls Assessed by Lula
+            include-controls:
+              - control-id: ID-1
+        description: Controls validated
+        remarks: Validation performed may indicate full or partial satisfaction
+      start: 2024-06-04T03:33:33.982591133Z
+      title: Lula Validation Result
+      uuid: ae611955-e97e-4115-a5d0-2e5ccbf35872
+    - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43
+      findings:
+        - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance.
+          related-observations:
+            - observation-uuid: 20d943cb-8a8d-47e2-b625-2d5829604f5e
+          target:
+            status:
+              state: satisfied
+            target-id: ID-1
+            type: objective-id
+          title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control:  ID-1'
+          uuid: fbee2fb0-e8ba-419c-9d29-f20525bbc420
+      observations:
+        - collected: 2024-06-04T02:48:06.890379789Z
+          description: |+
+            [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9
+
+          methods:
+            - TEST
+          relevant-evidence:
+            - description: |
+                Result: satisfied
+          uuid: 20d943cb-8a8d-47e2-b625-2d5829604f5e
+      props:
+        - name: threshold
+          ns: https://docs.lula.dev/ns
+          value: "false"
+      reviewed-controls:
+        control-selections:
+          - description: Controls Assessed by Lula
+            include-controls:
+              - control-id: ID-1
+        description: Controls validated
+        remarks: Validation performed may indicate full or partial satisfaction
+      start: 2024-06-04T02:48:06.895205436Z
+      title: Lula Validation Result
+      uuid: 7ef25ccd-22b8-421e-8c64-9fcbde697696
+    - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43
+      findings:
+        - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance.
+          related-observations:
+            - observation-uuid: d3b228e8-8c8b-4be9-a0b0-8da9fcdb005f
+          target:
+            status:
+              state: not-satisfied
+            target-id: ID-1
+            type: objective-id
+          title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control:  ID-1'
+          uuid: 44fc8e49-b633-4880-8d58-4ed0f14c7bdb
+      observations:
+        - collected: 2024-06-04T02:46:29.018416946Z
+          description: |+
+            [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9
+
+          methods:
+            - TEST
+          relevant-evidence:
+            - description: |
+                Result: not-satisfied
+          uuid: d3b228e8-8c8b-4be9-a0b0-8da9fcdb005f
+      props:
+        - name: threshold
+          ns: https://docs.lula.dev/ns
+          value: "false"
+      reviewed-controls:
+        control-selections:
+          - description: Controls Assessed by Lula
+            include-controls:
+              - control-id: ID-1
+        description: Controls validated
+        remarks: Validation performed may indicate full or partial satisfaction
+      start: 2024-06-04T02:46:29.02587111Z
+      title: Lula Validation Result
+      uuid: 250ac234-519f-4f26-bee3-98e870feeb7a
+  uuid: 9673739a-9379-4575-b454-1b42672a4970