feat(template)!: introducing variables and sensitive configuration

design-docs/senstive-config.md

@@ -0,0 +1,89 @@
+# Designing Sensitive Configuration
+
+Author(s): @meganwolf0
+Date Created: Sept 17, 2024
+Status: DRAFT
+Ticket: [#641](https://github.com/defenseunicorns/lula/issues/641)
+Reviews Requested By: TBD
+
+## Problem Statement
+
+Given that validation data may incorporate sensitive information (e.g., API Keys, passwords, etc.), we need to determine the solution for 
+1. Mapping those values to their source (e.g., environment variables, config files, secret store, etc.)
+    -> I think as an initial cut, this should be scoped to variables only coming from the environment, maybe include functionality to source .env files
+2. Ensuring that the values are masked in the output of the validation (e.g., when printing the component-definition)
+3. Having the ability to manually template the values, specifically in the case of testing the validation content (probably in the purview of `lula dev`)
+
+## Proposal
+
+Comparing a few different options:
+
+### Option 1: Go templates + string replacement
+Basically, establish a prefix for the variables that are sensitive, then perform a string repace for the templated values -> also can apply a masking function when running the output?
+
+(see https://github.com/defenseunicorns/lula/tree/517-go-template-testing for an example of templating something with "secret.xx")
+
+### Option 2: HCL variable templating
+I think this is probably generally more of a heavy-lift to get set-up, and will also introduce more verbosity when setting up variables, but could be a more flexible solution.
+
+This is actually probably more relevant with generally templating, as it handles complex data structures...
+
+## Scope and Requirements
+
+IMO, the best way to define the scope is to determine which examples we are trying to cover:
+
+1. Templating a sensitive API Key in a notional Domain API-Spec
+
+```yaml
+# ... rest of the validation ...
+domain:
+  type: api
+  api-spec:
+    requests:
+    - name: local
+      url: https://some.url/v1/api
+      content-type: application/json
+      headers:
+        Authorization: Bearer {{ .secret.api_key }}
+# ... rest of the validation ...
+```
+
+2. Templating a sensitive policy value...?
+
+```yaml
+# ...
+# ...
+```
+
+## Implementation Details
+
+Tiers of templating:
+
+* Lula Tools Template -> Default templates constants + variables
+
+Expand upon the implementation details here. Draw the reader’s attention to:
+
+* Changes to existing systems.
+* Creation of new systems.
+* Impacts to the customer.
+* Include code samples where possible.
+
+## Metrics & Alerts
+
+List any Metrics / Alerts that you plan to include in the system design
+
+## Alternatives Considered
+
+List any alternative solutions considered and how the proposed solution is a better fit.
+
+## Non-Goals
+
+List out anything that may be related to the solution, but won’t be covered by this solution.
+
+## Future Improvements:
+
+List out anything that won’t be included in this version of the feature/solution, but could be revisited or iterated upon in the future.
+
+## Other Considerations:
+
+List anything else that won’t be solved by the solution

lula-config.yaml

@@ -1 +1,35 @@
-log_level: info
+
+# constants = place to define non-changing values that can be of any type
+# I think stuff here probably shouldn't be set by env vars - it's hard to be deterministic because of the character set differences, also type differences could lead to weird side effects
+# Another note about this - we could probably easily pull in values of child components if this was referenced from a system-level - so this kind of behaves a bit like help values.yaml
+constants:
+  # map[string]interface{} - elements referenced via template as {{ .const.key }}
+  type: software
+  title: lula
+  # Sample: Istio-specific values
+  istio:
+    namespace: istio-system # overriden by --set const.istio.namespace=my-istio-namespace
+  resources:
+    jsoncm: configmaps # (NOT) overriden by LULA_VAR_RESOURCES_JSONCM
+    # Problem with this is that json-cm and json_cm are different yaml keys, but would possibly reconcile to the same thing... so you're getting some side effects here that aren't great.
+    yamlcm: configmaps
+    secret: secrets
+    pod: pods
+    boolean: false  # (NOT) overriden by LULA_VAR_RESOURCES_BOOLEAN 
+    # ok how does this work when they're different types? an env var will always be a string...
+  exemptions:
+    - one
+    - two
+    - three
+
+# variables = place to define changing values of string type, and optionally sensitive values
+variables:
+  - key: some_lula_secret # set by LULA_VAR_SOME_LULA_SECRET / overriden by --set var.some_lula_secret=my-secret
+    default: blahblah  # optional
+    sensitive: true # {{ var.some_lula_secret | mask }}
+  - key: some_env_var
+    default: this-should-be-overridden
+
+# Lula config values, still accessible via LULA_*, where * is the key
+log_level: info
+target: il5

src/cmd/common/viper.go

@@ -10,9 +10,11 @@ import (
 )
 
 const (
-	VLogLevel = "log_level"
-	VTarget   = "target"
-	VSummary  = "summary"
+	VLogLevel  = "log_level"
+	VTarget    = "target"
+	VSummary   = "summary"
+	VConstants = "constants"
+	VVariables = "variables"
 )
 
 var (
@@ -78,6 +80,8 @@ func isVersionCmd() bool {
 func setDefaults() {
 	v.SetDefault(VLogLevel, "info")
 	v.SetDefault(VSummary, false)
+	v.SetDefault(VConstants, make(map[string]interface{}))
+	v.SetDefault(VVariables, make([]interface{}, 0))
 }
 
 func printViperConfigUsed() {

src/cmd/tools/template.go

@@ -1,7 +1,9 @@
 package tools
 
 import (
+	"fmt"
 	"os"
+	"strings"
 
 	"github.com/defenseunicorns/go-oscal/src/pkg/files"
 	"github.com/defenseunicorns/lula/src/cmd/common"
@@ -12,8 +14,14 @@ import (
 )
 
 type templateFlags struct {
-	InputFile  string // -f --input-file
-	OutputFile string // -o --output-file
+	InputFile  string   // -f --input-file
+	OutputFile string   // -o --output-file
+	Set        []string // --set
+	All        bool     // --all
+	// some demo flags
+	Const        bool // --const
+	NonSensitive bool // --non-sensitive
+	Sensitive    bool // --sensitive
 }
 
 var templateOpts = &templateFlags{}
@@ -25,7 +33,14 @@ To template an OSCAL Model:
 To indicate a specific output file:
 	lula tools template -f ./oscal-component.yaml -o templated-oscal-component.yaml
 
-Data for the templating should be stored under the 'variables' configuration item in a lula-config.yaml file
+To perform overrides on the template data:
+	lula tools template -f ./oscal-component.yaml --set var.key1=value1 --set const.key2=value2
+
+To perform the full template operation, including sensitive data:
+	lula tools template -f ./oscal-component.yaml --all
+
+Data for templating should be stored under 'constants' or 'variables' configuration items in a lula-config.yaml file
+See documentation for more detail on configuration schema
 `
 var templateCmd = &cobra.Command{
 	Use:     "template",
@@ -42,17 +57,54 @@ var templateCmd = &cobra.Command{
 
 		// Get current viper pointer
 		v := common.GetViper()
-		// Get all viper settings
-		// This will only return config file items and resolved environment variables
-		// that have an associated key in the config file
-		viperData := v.AllSettings()
 
-		// Handles merging viper config file data + environment variables
-		mergedMap := template.CollectTemplatingData(viperData)
+		// Get constants and variables for templating from viper config
+		var constants map[string]interface{}
+		var variables []template.VariableConfig
 
-		templatedData, err := template.ExecuteTemplate(mergedMap, string(data))
+		err = v.UnmarshalKey(common.VConstants, &constants)
 		if err != nil {
-			message.Fatalf(err, "error templating validation: %v", err)
+			message.Fatalf(err, "unable to unmarshal constants into map: %v", err)
+		}
+
+		err = v.UnmarshalKey(common.VVariables, &variables)
+		if err != nil {
+			message.Fatalf(err, "unable to unmarshal variables into slice: %v", err)
+		}
+
+		// Handles merging viper config file data + environment variables
+		templateData := template.CollectTemplatingData(constants, variables)
+
+		// override anything that's --set
+		overrideTemplateValues(templateData, templateOpts.Set)
+
+		// Execute the template function based on the flags (TESTING ONLY)
+		var templatedData []byte
+		if templateOpts.Const {
+			templatedData, err = template.ExecuteConstTemplate(templateData.Constants, string(data))
+			if err != nil {
+				message.Fatalf(err, "error templating validation: %v", err)
+			}
+		} else if templateOpts.NonSensitive {
+			templatedData, err = template.ExecuteNonSensitiveTemplate(templateData, string(data))
+			if err != nil {
+				message.Fatalf(err, "error templating validation: %v", err)
+			}
+		} else if templateOpts.Sensitive {
+			templatedData, err = template.ExecuteSensitiveTemplate(templateData, string(data))
+			if err != nil {
+				message.Fatalf(err, "error templating validation: %v", err)
+			}
+		} else if templateOpts.All {
+			templatedData, err = template.ExecuteFullTemplate(templateData, string(data))
+			if err != nil {
+				message.Fatalf(err, "error templating validation: %v", err)
+			}
+		} else {
+			templatedData, err = template.ExecuteMaskedTemplate(templateData, string(data))
+			if err != nil {
+				message.Fatalf(err, "error templating validation: %v", err)
+			}
 		}
 
 		if templateOpts.OutputFile == "" {
@@ -86,4 +138,55 @@ func init() {
 	templateCmd.Flags().StringVarP(&templateOpts.InputFile, "input-file", "f", "", "the path to the target artifact")
 	templateCmd.MarkFlagRequired("input-file")
 	templateCmd.Flags().StringVarP(&templateOpts.OutputFile, "output-file", "o", "", "the path to the output file. If not specified, the output file will be directed to stdout")
+	templateCmd.Flags().StringSliceVarP(&templateOpts.Set, "set", "s", []string{}, "set a value in the template data")
+
+	templateCmd.Flags().BoolVar(&templateOpts.Const, "const", false, "only include constants in the template")
+	templateCmd.Flags().BoolVar(&templateOpts.NonSensitive, "non-sensitive", false, "only include non-sensitive variables in the template")
+	templateCmd.Flags().BoolVar(&templateOpts.Sensitive, "sensitive", false, "only include sensitive variables in the template")
+	templateCmd.Flags().BoolVar(&templateOpts.All, "all", false, "include all variables in the template")
+}
+
+func overrideTemplateValues(templateData *template.TemplateData, setFlags []string) {
+	for _, flag := range setFlags {
+		parts := strings.SplitN(flag, "=", 2)
+		if len(parts) != 2 {
+			message.Fatalf(fmt.Errorf("invalid --set flag format, should be key.path=value"), "invalid --set flag format, should be key.path=value")
+		}
+		path, value := parts[0], parts[1]
+
+		// for each set flag, check if .var or .const
+		// if .var, set the value in the templateData.Variables
+		// if .const, set the value in the templateData.Constants
+		if strings.HasPrefix(path, "."+template.VAR+".") {
+			// Set the value in the templateData.Variables
+			key := strings.TrimPrefix(path, "."+template.VAR+".")
+			templateData.Variables[key] = value
+		} else if strings.HasPrefix(path, "."+template.CONST+".") {
+			// Set the value in the templateData.Constants
+			key := strings.TrimPrefix(path, "."+template.CONST+".")
+			setNestedValue(templateData.Constants, key, value)
+		}
+	}
+}
+
+// Helper function to set a value in a map based on a JSON-like key path
+func setNestedValue(m map[string]interface{}, path string, value interface{}) error {
+	keys := strings.Split(path, ".")
+	lastKey := keys[len(keys)-1]
+
+	// Traverse the map, creating intermediate maps if necessary
+	for _, key := range keys[:len(keys)-1] {
+		if _, exists := m[key]; !exists {
+			m[key] = make(map[string]interface{})
+		}
+		if nestedMap, ok := m[key].(map[string]interface{}); ok {
+			m = nestedMap
+		} else {
+			return fmt.Errorf("path %s contains a non-map value", key)
+		}
+	}
+
+	// Set the final value
+	m[lastKey] = value
+	return nil
 }

src/internal/template/helpers.go

@@ -0,0 +1,14 @@
+package template
+
+import "fmt"
+
+func concatToRegoList(list []any) string {
+	regoList := ""
+	for i, item := range list {
+		if i > 0 {
+			regoList += ", "
+		}
+		regoList += `"` + fmt.Sprintf("%v", item) + `"`
+	}
+	return regoList
+}