From 0fe5ba1c085d6cfe49d73d55540de84f24d29287 Mon Sep 17 00:00:00 2001 From: Megan Wolf Date: Wed, 18 Sep 2024 13:31:32 -0400 Subject: [PATCH 1/9] feat(template): sample sensitive config --- design-docs/senstive-config.md | 89 ++++++++++++ lula-config.yaml | 36 ++++- src/cmd/common/viper.go | 10 +- src/cmd/tools/template.go | 125 +++++++++++++++-- src/internal/template/helpers.go | 14 ++ src/internal/template/template.go | 186 +++++++++++++++++++++++-- src/internal/template/template_test.go | 73 ++++++---- src/internal/template/types.go | 21 +++ src/test/e2e/standard/lula-config.yaml | 41 ++++-- src/test/e2e/standard/template_test.go | 6 + src/test/e2e/standard/validation.tpl | 30 ++++ 11 files changed, 569 insertions(+), 62 deletions(-) create mode 100644 design-docs/senstive-config.md create mode 100644 src/internal/template/helpers.go create mode 100644 src/internal/template/types.go create mode 100644 src/test/e2e/standard/validation.tpl diff --git a/design-docs/senstive-config.md b/design-docs/senstive-config.md new file mode 100644 index 00000000..2fdbc70c --- /dev/null +++ b/design-docs/senstive-config.md @@ -0,0 +1,89 @@ +# Designing Sensitive Configuration + +Author(s): @meganwolf0 +Date Created: Sept 17, 2024 +Status: DRAFT +Ticket: [#641](https://github.com/defenseunicorns/lula/issues/641) +Reviews Requested By: TBD + +## Problem Statement + +Given that validation data may incorporate sensitive information (e.g., API Keys, passwords, etc.), we need to determine the solution for +1. Mapping those values to their source (e.g., environment variables, config files, secret store, etc.) + -> I think as an initial cut, this should be scoped to variables only coming from the environment, maybe include functionality to source .env files +2. Ensuring that the values are masked in the output of the validation (e.g., when printing the component-definition) +3. Having the ability to manually template the values, specifically in the case of testing the validation content (probably in the purview of `lula dev`) + +## Proposal + +Comparing a few different options: + +### Option 1: Go templates + string replacement +Basically, establish a prefix for the variables that are sensitive, then perform a string repace for the templated values -> also can apply a masking function when running the output? + +(see https://github.com/defenseunicorns/lula/tree/517-go-template-testing for an example of templating something with "secret.xx") + +### Option 2: HCL variable templating +I think this is probably generally more of a heavy-lift to get set-up, and will also introduce more verbosity when setting up variables, but could be a more flexible solution. + +This is actually probably more relevant with generally templating, as it handles complex data structures... + +## Scope and Requirements + +IMO, the best way to define the scope is to determine which examples we are trying to cover: + +1. Templating a sensitive API Key in a notional Domain API-Spec + +```yaml +# ... rest of the validation ... +domain: + type: api + api-spec: + requests: + - name: local + url: https://some.url/v1/api + content-type: application/json + headers: + Authorization: Bearer {{ .secret.api_key }} +# ... rest of the validation ... +``` + +2. Templating a sensitive policy value...? + +```yaml +# ... +# ... +``` + +## Implementation Details + +Tiers of templating: + +* Lula Tools Template -> Default templates constants + variables + +Expand upon the implementation details here. Draw the reader’s attention to: + +* Changes to existing systems. +* Creation of new systems. +* Impacts to the customer. +* Include code samples where possible. + +## Metrics & Alerts + +List any Metrics / Alerts that you plan to include in the system design + +## Alternatives Considered + +List any alternative solutions considered and how the proposed solution is a better fit. + +## Non-Goals + +List out anything that may be related to the solution, but won’t be covered by this solution. + +## Future Improvements: + +List out anything that won’t be included in this version of the feature/solution, but could be revisited or iterated upon in the future. + +## Other Considerations: + +List anything else that won’t be solved by the solution \ No newline at end of file diff --git a/lula-config.yaml b/lula-config.yaml index da38ff76..e1c84b3b 100644 --- a/lula-config.yaml +++ b/lula-config.yaml @@ -1 +1,35 @@ -log_level: info \ No newline at end of file + +# constants = place to define non-changing values that can be of any type +# I think stuff here probably shouldn't be set by env vars - it's hard to be deterministic because of the character set differences, also type differences could lead to weird side effects +# Another note about this - we could probably easily pull in values of child components if this was referenced from a system-level - so this kind of behaves a bit like help values.yaml +constants: + # map[string]interface{} - elements referenced via template as {{ .const.key }} + type: software + title: lula + # Sample: Istio-specific values + istio: + namespace: istio-system # overriden by --set const.istio.namespace=my-istio-namespace + resources: + jsoncm: configmaps # (NOT) overriden by LULA_VAR_RESOURCES_JSONCM + # Problem with this is that json-cm and json_cm are different yaml keys, but would possibly reconcile to the same thing... so you're getting some side effects here that aren't great. + yamlcm: configmaps + secret: secrets + pod: pods + boolean: false # (NOT) overriden by LULA_VAR_RESOURCES_BOOLEAN + # ok how does this work when they're different types? an env var will always be a string... + exemptions: + - one + - two + - three + +# variables = place to define changing values of string type, and optionally sensitive values +variables: + - key: some_lula_secret # set by LULA_VAR_SOME_LULA_SECRET / overriden by --set var.some_lula_secret=my-secret + default: blahblah # optional + sensitive: true # {{ var.some_lula_secret | mask }} + - key: some_env_var + default: this-should-be-overridden + +# Lula config values, still accessible via LULA_*, where * is the key +log_level: info +target: il5 diff --git a/src/cmd/common/viper.go b/src/cmd/common/viper.go index 7ebe0034..7d3ea027 100644 --- a/src/cmd/common/viper.go +++ b/src/cmd/common/viper.go @@ -10,9 +10,11 @@ import ( ) const ( - VLogLevel = "log_level" - VTarget = "target" - VSummary = "summary" + VLogLevel = "log_level" + VTarget = "target" + VSummary = "summary" + VConstants = "constants" + VVariables = "variables" ) var ( @@ -78,6 +80,8 @@ func isVersionCmd() bool { func setDefaults() { v.SetDefault(VLogLevel, "info") v.SetDefault(VSummary, false) + v.SetDefault(VConstants, make(map[string]interface{})) + v.SetDefault(VVariables, make([]interface{}, 0)) } func printViperConfigUsed() { diff --git a/src/cmd/tools/template.go b/src/cmd/tools/template.go index e9b6f5ae..e38114d1 100644 --- a/src/cmd/tools/template.go +++ b/src/cmd/tools/template.go @@ -1,7 +1,9 @@ package tools import ( + "fmt" "os" + "strings" "github.com/defenseunicorns/go-oscal/src/pkg/files" "github.com/defenseunicorns/lula/src/cmd/common" @@ -12,8 +14,14 @@ import ( ) type templateFlags struct { - InputFile string // -f --input-file - OutputFile string // -o --output-file + InputFile string // -f --input-file + OutputFile string // -o --output-file + Set []string // --set + All bool // --all + // some demo flags + Const bool // --const + NonSensitive bool // --non-sensitive + Sensitive bool // --sensitive } var templateOpts = &templateFlags{} @@ -25,7 +33,14 @@ To template an OSCAL Model: To indicate a specific output file: lula tools template -f ./oscal-component.yaml -o templated-oscal-component.yaml -Data for the templating should be stored under the 'variables' configuration item in a lula-config.yaml file +To perform overrides on the template data: + lula tools template -f ./oscal-component.yaml --set var.key1=value1 --set const.key2=value2 + +To perform the full template operation, including sensitive data: + lula tools template -f ./oscal-component.yaml --all + +Data for templating should be stored under 'constants' or 'variables' configuration items in a lula-config.yaml file +See documentation for more detail on configuration schema ` var templateCmd = &cobra.Command{ Use: "template", @@ -42,17 +57,54 @@ var templateCmd = &cobra.Command{ // Get current viper pointer v := common.GetViper() - // Get all viper settings - // This will only return config file items and resolved environment variables - // that have an associated key in the config file - viperData := v.AllSettings() - // Handles merging viper config file data + environment variables - mergedMap := template.CollectTemplatingData(viperData) + // Get constants and variables for templating from viper config + var constants map[string]interface{} + var variables []template.VariableConfig - templatedData, err := template.ExecuteTemplate(mergedMap, string(data)) + err = v.UnmarshalKey(common.VConstants, &constants) if err != nil { - message.Fatalf(err, "error templating validation: %v", err) + message.Fatalf(err, "unable to unmarshal constants into map: %v", err) + } + + err = v.UnmarshalKey(common.VVariables, &variables) + if err != nil { + message.Fatalf(err, "unable to unmarshal variables into slice: %v", err) + } + + // Handles merging viper config file data + environment variables + templateData := template.CollectTemplatingData(constants, variables) + + // override anything that's --set + overrideTemplateValues(templateData, templateOpts.Set) + + // Execute the template function based on the flags (TESTING ONLY) + var templatedData []byte + if templateOpts.Const { + templatedData, err = template.ExecuteConstTemplate(templateData.Constants, string(data)) + if err != nil { + message.Fatalf(err, "error templating validation: %v", err) + } + } else if templateOpts.NonSensitive { + templatedData, err = template.ExecuteNonSensitiveTemplate(templateData, string(data)) + if err != nil { + message.Fatalf(err, "error templating validation: %v", err) + } + } else if templateOpts.Sensitive { + templatedData, err = template.ExecuteSensitiveTemplate(templateData, string(data)) + if err != nil { + message.Fatalf(err, "error templating validation: %v", err) + } + } else if templateOpts.All { + templatedData, err = template.ExecuteFullTemplate(templateData, string(data)) + if err != nil { + message.Fatalf(err, "error templating validation: %v", err) + } + } else { + templatedData, err = template.ExecuteMaskedTemplate(templateData, string(data)) + if err != nil { + message.Fatalf(err, "error templating validation: %v", err) + } } if templateOpts.OutputFile == "" { @@ -86,4 +138,55 @@ func init() { templateCmd.Flags().StringVarP(&templateOpts.InputFile, "input-file", "f", "", "the path to the target artifact") templateCmd.MarkFlagRequired("input-file") templateCmd.Flags().StringVarP(&templateOpts.OutputFile, "output-file", "o", "", "the path to the output file. If not specified, the output file will be directed to stdout") + templateCmd.Flags().StringSliceVarP(&templateOpts.Set, "set", "s", []string{}, "set a value in the template data") + + templateCmd.Flags().BoolVar(&templateOpts.Const, "const", false, "only include constants in the template") + templateCmd.Flags().BoolVar(&templateOpts.NonSensitive, "non-sensitive", false, "only include non-sensitive variables in the template") + templateCmd.Flags().BoolVar(&templateOpts.Sensitive, "sensitive", false, "only include sensitive variables in the template") + templateCmd.Flags().BoolVar(&templateOpts.All, "all", false, "include all variables in the template") +} + +func overrideTemplateValues(templateData *template.TemplateData, setFlags []string) { + for _, flag := range setFlags { + parts := strings.SplitN(flag, "=", 2) + if len(parts) != 2 { + message.Fatalf(fmt.Errorf("invalid --set flag format, should be key.path=value"), "invalid --set flag format, should be key.path=value") + } + path, value := parts[0], parts[1] + + // for each set flag, check if .var or .const + // if .var, set the value in the templateData.Variables + // if .const, set the value in the templateData.Constants + if strings.HasPrefix(path, "."+template.VAR+".") { + // Set the value in the templateData.Variables + key := strings.TrimPrefix(path, "."+template.VAR+".") + templateData.Variables[key] = value + } else if strings.HasPrefix(path, "."+template.CONST+".") { + // Set the value in the templateData.Constants + key := strings.TrimPrefix(path, "."+template.CONST+".") + setNestedValue(templateData.Constants, key, value) + } + } +} + +// Helper function to set a value in a map based on a JSON-like key path +func setNestedValue(m map[string]interface{}, path string, value interface{}) error { + keys := strings.Split(path, ".") + lastKey := keys[len(keys)-1] + + // Traverse the map, creating intermediate maps if necessary + for _, key := range keys[:len(keys)-1] { + if _, exists := m[key]; !exists { + m[key] = make(map[string]interface{}) + } + if nestedMap, ok := m[key].(map[string]interface{}); ok { + m = nestedMap + } else { + return fmt.Errorf("path %s contains a non-map value", key) + } + } + + // Set the final value + m[lastKey] = value + return nil } diff --git a/src/internal/template/helpers.go b/src/internal/template/helpers.go new file mode 100644 index 00000000..291d226f --- /dev/null +++ b/src/internal/template/helpers.go @@ -0,0 +1,14 @@ +package template + +import "fmt" + +func concatToRegoList(list []any) string { + regoList := "" + for i, item := range list { + if i > 0 { + regoList += ", " + } + regoList += `"` + fmt.Sprintf("%v", item) + `"` + } + return regoList +} diff --git a/src/internal/template/template.go b/src/internal/template/template.go index a67ccd2f..0fe60989 100644 --- a/src/internal/template/template.go +++ b/src/internal/template/template.go @@ -2,24 +2,120 @@ package template import ( "os" + "regexp" "strings" "text/template" +) - "github.com/defenseunicorns/pkg/helpers" +const ( + PREFIX = "LULA_VAR_" + CONST = "const" + VAR = "var" ) -const PREFIX = "LULA_" +func createTemplate() *template.Template { + // Register custom template functions + funcMap := template.FuncMap{ + "concatToRegoList": func(a []any) string { + return concatToRegoList(a) + }, + "mask": func(a string) string { + return "********" + }, + // Add more custom functions as needed + } + + // Parse the template and apply the function map + tpl := template.New("template").Funcs(funcMap) + tpl.Option("missingkey=zero") + return tpl +} + +// ExecuteFullTemplate templates everything +func ExecuteFullTemplate(templateData *TemplateData, templateString string) ([]byte, error) { + tpl := createTemplate() + tpl, err := tpl.Parse(templateString) + if err != nil { + return []byte{}, err + } + + var buffer strings.Builder + allVars := MergeStringMaps(templateData.Variables, templateData.SensitiveVariables) + err = tpl.Execute(&buffer, map[string]interface{}{ + CONST: templateData.Constants, + VAR: allVars}) + if err != nil { + return []byte{}, err + } + + return []byte(buffer.String()), nil +} + +// ExecuteConstTemplate templates only constants +// this templates only values in the constants map +func ExecuteConstTemplate(constants map[string]interface{}, templateString string) ([]byte, error) { + // Find anything {{ var.KEY }} and replace with {{ "{{ var.KEY }}" }} + re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) + templateString = re.ReplaceAllString(templateString, "{{ \"{{ ."+VAR+".$1 }}\" }}") + + tpl := createTemplate() + tpl, err := tpl.Parse(templateString) + if err != nil { + return []byte{}, err + } + + var buffer strings.Builder + err = tpl.Execute(&buffer, map[string]interface{}{ + CONST: constants}) + if err != nil { + return []byte{}, err + } + + return []byte(buffer.String()), nil +} + +// ExecuteNonSensitiveTemplate templates only constants and non-sensitive variables +// used for compose operations +func ExecuteNonSensitiveTemplate(templateData *TemplateData, templateString string) ([]byte, error) { + // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ "{{ var.KEY }}" }} + re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) + varMatches := re.FindStringSubmatch(templateString) + for _, m := range varMatches { + if _, ok := templateData.SensitiveVariables[m]; ok { + reSensitive := regexp.MustCompile(`{{\s*\.` + VAR + `\.` + m + `\s*}}`) + templateString = reSensitive.ReplaceAllString(templateString, "{{ \"{{ ."+VAR+"."+m+" }}\" }}") + } + } + + tpl := createTemplate() + tpl, err := tpl.Parse(templateString) + if err != nil { + return []byte{}, err + } + + var buffer strings.Builder + err = tpl.Execute(&buffer, map[string]interface{}{ + CONST: templateData.Constants, + VAR: templateData.Variables}) + if err != nil { + return []byte{}, err + } -// ExecuteTemplate templates the template string with the data map -func ExecuteTemplate(data map[string]interface{}, templateString string) ([]byte, error) { - tmpl, err := template.New("template").Parse(templateString) + return []byte(buffer.String()), nil +} + +// ExecuteSensitiveTemplate templates the sensitive variables +// for use immediately before validation, after non-sensitive data is templated, results should not be written +func ExecuteSensitiveTemplate(templateData *TemplateData, templateString string) ([]byte, error) { + tpl := createTemplate() + tpl, err := tpl.Parse(templateString) if err != nil { return []byte{}, err } - tmpl.Option("missingkey=default") var buffer strings.Builder - err = tmpl.Execute(&buffer, data) + err = tpl.Execute(&buffer, map[string]interface{}{ + VAR: templateData.SensitiveVariables}) if err != nil { return []byte{}, err } @@ -27,23 +123,67 @@ func ExecuteTemplate(data map[string]interface{}, templateString string) ([]byte return []byte(buffer.String()), nil } -// Prepare the map of data for use in templating +// ExecuteMaskedTemplate templates all values, but masks the sensitive ones +// for display/printing only +func ExecuteMaskedTemplate(templateData *TemplateData, templateString string) ([]byte, error) { + // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ var.KEY | mask }} + re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) + varMatches := re.FindStringSubmatch(templateString) + for _, m := range varMatches { + if _, ok := templateData.SensitiveVariables[m]; ok { + reSensitive := regexp.MustCompile(`{{\s*\.` + VAR + `\.` + m + `\s*}}`) + templateString = reSensitive.ReplaceAllString(templateString, "{{ ."+VAR+"."+m+" | mask }}") + } + } + + tpl := createTemplate() + tpl, err := tpl.Parse(templateString) + if err != nil { + return []byte{}, err + } + + var buffer strings.Builder + allVars := MergeStringMaps(templateData.Variables, templateData.SensitiveVariables) + err = tpl.Execute(&buffer, map[string]interface{}{ + CONST: templateData.Constants, + VAR: allVars}) + if err != nil { + return []byte{}, err + } -func CollectTemplatingData(data map[string]interface{}) map[string]interface{} { + return []byte(buffer.String()), nil +} + +// Prepare the templateData object for use in templating +func CollectTemplatingData(constants map[string]interface{}, variables []VariableConfig) *TemplateData { + // Create the TemplateData object from the constants and variables + templateData := NewTemplateData() + templateData.Constants = constants + for _, variable := range variables { + // convert '-' to '_' in the key and remove any special characters + variable.Key = strings.ReplaceAll(variable.Key, "-", "_") + re := regexp.MustCompile(`[^a-zA-Z0-9_]`) + variable.Key = re.ReplaceAllString(variable.Key, "") + + templateData.Variables[variable.Key] = variable.Default + if variable.Sensitive { + templateData.SensitiveVariables[variable.Key] = variable.Default + } + } // Get all environment variables with a specific prefix envMap := GetEnvVars(PREFIX) - // Merge the data into a single map for use with templating - mergedMap := helpers.MergeMapRecursive(envMap, data) - - return mergedMap + // Update the templateData with the environment variables overrides + templateData.Variables = MergeStringMaps(templateData.Variables, envMap) + templateData.SensitiveVariables = MergeStringMaps(templateData.SensitiveVariables, envMap) + return templateData } // get all environment variables with the established prefix -func GetEnvVars(prefix string) map[string]interface{} { - envMap := make(map[string]interface{}) +func GetEnvVars(prefix string) map[string]string { + envMap := make(map[string]string) // Get all environment variables envVars := os.Environ() @@ -69,3 +209,19 @@ func GetEnvVars(prefix string) map[string]interface{} { return envMap } + +// MergeStringMaps merges two maps of strings into a single map of strings. +// m2 will overwrite m1 if a key exists in both maps. +func MergeStringMaps(m1, m2 map[string]string) map[string]string { + r := map[string]string{} + + for key, value := range m1 { + r[key] = value + } + + for key, value := range m2 { + r[key] = value + } + + return r +} diff --git a/src/internal/template/template_test.go b/src/internal/template/template_test.go index 379ebe9f..d922fcf2 100644 --- a/src/internal/template/template_test.go +++ b/src/internal/template/template_test.go @@ -2,49 +2,50 @@ package template_test import ( "os" + "reflect" "strings" "testing" "github.com/defenseunicorns/lula/src/internal/template" ) -func TestExecuteTemplate(t *testing.T) { +// func TestExecuteTemplate(t *testing.T) { - test := func(t *testing.T, data map[string]interface{}, preTemplate string, expected string) { - t.Helper() - // templateData returned - got, err := template.ExecuteTemplate(data, preTemplate) - if err != nil { - t.Fatalf("error templating data: %s\n", err.Error()) - } +// test := func(t *testing.T, data map[string]interface{}, preTemplate string, expected string) { +// t.Helper() +// // templateData returned +// got, err := template.ExecuteTemplate(data, preTemplate) +// if err != nil { +// t.Fatalf("error templating data: %s\n", err.Error()) +// } - if string(got) != expected { - t.Fatalf("Expected %s - Got %s\n", expected, string(got)) - } - } +// if string(got) != expected { +// t.Fatalf("Expected %s - Got %s\n", expected, string(got)) +// } +// } - t.Run("Test {{ .testVar }} with data", func(t *testing.T) { - data := map[string]interface{}{ - "testVar": "testing", - } +// t.Run("Test {{ .testVar }} with data", func(t *testing.T) { +// data := map[string]interface{}{ +// "testVar": "testing", +// } - test(t, data, "{{ .testVar }}", "testing") - }) +// test(t, data, "{{ .testVar }}", "testing") +// }) - t.Run("Test {{ .testVar }} but empty data", func(t *testing.T) { - data := map[string]interface{}{} +// t.Run("Test {{ .testVar }} but empty data", func(t *testing.T) { +// data := map[string]interface{}{} - test(t, data, "{{ .testVar }}", "") - }) +// test(t, data, "{{ .testVar }}", "") +// }) -} +// } func TestGetEnvVars(t *testing.T) { test := func(t *testing.T, prefix string, key string, value string) { t.Helper() - os.Setenv(key, value) + // os.Setenv(key, value) envMap := template.GetEnvVars(prefix) // convert key to expected format @@ -64,3 +65,27 @@ func TestGetEnvVars(t *testing.T) { test(t, "OTHER_", "OTHER_RESOURCE", "deployments") }) } + +func TestMergeStringMaps(t *testing.T) { + t.Parallel() // Enable parallel execution of tests + t.Run("Should merge two maps", func(t *testing.T) { + t.Parallel() // Enable parallel execution of subtests + m1 := map[string]string{ + "key1": "value1", + "key2": "value2", + } + m2 := map[string]string{ + "key1": "value3", + "key3": "value4", + } + expected := map[string]string{ + "key1": "value3", + "key2": "value2", + "key3": "value4", + } + result := template.MergeStringMaps(m1, m2) + if !reflect.DeepEqual(result, expected) { + t.Errorf("MergeStringMaps() got = %v, want %v", result, expected) + } + }) +} diff --git a/src/internal/template/types.go b/src/internal/template/types.go new file mode 100644 index 00000000..84dd4d78 --- /dev/null +++ b/src/internal/template/types.go @@ -0,0 +1,21 @@ +package template + +type VariableConfig struct { + Key string + Default string + Sensitive bool +} + +type TemplateData struct { + Constants map[string]interface{} + Variables map[string]string + SensitiveVariables map[string]string +} + +func NewTemplateData() *TemplateData { + return &TemplateData{ + Constants: make(map[string]interface{}), + Variables: make(map[string]string), + SensitiveVariables: make(map[string]string), + } +} diff --git a/src/test/e2e/standard/lula-config.yaml b/src/test/e2e/standard/lula-config.yaml index 70e8fe6f..92a84fdb 100644 --- a/src/test/e2e/standard/lula-config.yaml +++ b/src/test/e2e/standard/lula-config.yaml @@ -1,8 +1,33 @@ -resources: - jsoncm: configmaps - yamlcm: configmaps - secret: secrets - pod: pods - -type: software -title: lula \ No newline at end of file + +# If we had remote variables, e.g., co-located with the oscal data, are those automatically pulled in? and what is the override mechanism? Probably the system overrides the components... really this is just like re-hashing helm charts with values.yaml... +# If we wanted to move the validations to be CRDs at some point, this could give us a leg-up... + +# I think stuff here maybe shouldn't be set by env vars - as described it's hard to be deterministic +constants: +# since this field can take anything, even list data, there's not really a good mechanism for validating the data + type: software + title: lula + # Example: Istio-specific values + istio: + namespace: istio-system # overriden by --set const.istio.namespace=my-istio-namespace + resources: + jsoncm: configmaps # overriden by LULA_VAR_RESOURCES_JSONCM + # Problem with this is that json-cm and json_cm are different yaml keys, but would possibly reconcile to the same thing... so you're getting some side effects here that aren't great. + yamlcm: configmaps + secret: secrets + pod: pods + boolean: false # overriden by LULA_VAR_RESOURCES_BOOLEAN + # ok how does this work when they're different types? an env var will always be a string... +# how to handle "set" + +# string only variables, basically just from env vars +# the way zarf does this is they have to be defined here to be available, I believe +variables: + - key: some_lula_secret # LULA_VAR_SOME_LULA_SECRET? # overriden by --set var.some_lula_secret=my-secret + default: blahblah # optional + sensitive: true # {{ var.some_lula_secret }} -> look-up + - key: some_env_var + +# Lula config values +log_level: info +target: il5 diff --git a/src/test/e2e/standard/template_test.go b/src/test/e2e/standard/template_test.go index 478a8424..4fc25786 100644 --- a/src/test/e2e/standard/template_test.go +++ b/src/test/e2e/standard/template_test.go @@ -73,3 +73,9 @@ func TestTemplateCommand(t *testing.T) { // }) } + +// TODO: +// 1. Test --set for variables +// 2. Test with env vars + config vars +// 3. Test a type mismatch +// 4. Test secrets diff --git a/src/test/e2e/standard/validation.tpl b/src/test/e2e/standard/validation.tpl new file mode 100644 index 00000000..ff4a7dc0 --- /dev/null +++ b/src/test/e2e/standard/validation.tpl @@ -0,0 +1,30 @@ +domain: + type: api + api-spec: + requests: + - name: local + url: https://some.url/v1/api + content-type: application/json + headers: + Authorization: Bearer {{ .var.some_lula_secret }} +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + input.jsoncm.name == {{ .const.resources.jsoncm }} + input.yamlcm.logging.level == {{ .const.resources.yamlcm }} + not input.secret.name in { {{ .const.exemptions | concatToRegoList }} } # tmpl creates: "one", "two", "three" + } + msg = validate.msg + + test_env_var := {{ .var.some_env_var }} # non-sensitive + test_another_env_var := {{ .var.another_env_var }} # non-sensitive, no default \ No newline at end of file From 6bebbefa508e892fcaab050077aa04c143b98fca Mon Sep 17 00:00:00 2001 From: Megan Wolf Date: Thu, 19 Sep 2024 20:59:20 -0400 Subject: [PATCH 2/9] feat(template): updated funcs, tests --- design-docs/senstive-config.md | 89 --- docs/reference/configuration.md | 42 ++ lula-config.yaml | 36 +- src/cmd/tools/template.go | 224 +++--- src/internal/template/template.go | 333 +++++++-- src/internal/template/template_test.go | 657 ++++++++++++++++-- src/internal/template/types.go | 21 - src/test/e2e/standard/lula-config.yaml | 33 +- src/test/e2e/standard/template_test.go | 109 ++- src/test/e2e/standard/testdata/empty.golden | 0 src/test/e2e/standard/testdata/help.golden | 32 + .../e2e/standard/testdata/validation.golden | 30 + .../standard/testdata/validation_all.golden | 30 + .../testdata/validation_constants.golden | 30 + .../testdata/validation_non_sensitive.golden | 30 + .../testdata/validation_with_env_vars.golden | 30 + .../testdata/validation_with_set.golden | 30 + src/test/e2e/standard/validation.tpl | 30 - .../validation/validation.bad.tmpl.yaml | 30 + .../common/validation/validation.tmpl.yaml | 30 + src/test/util/utils.go | 9 +- 21 files changed, 1365 insertions(+), 490 deletions(-) delete mode 100644 design-docs/senstive-config.md delete mode 100644 src/internal/template/types.go create mode 100644 src/test/e2e/standard/testdata/empty.golden create mode 100644 src/test/e2e/standard/testdata/help.golden create mode 100644 src/test/e2e/standard/testdata/validation.golden create mode 100644 src/test/e2e/standard/testdata/validation_all.golden create mode 100644 src/test/e2e/standard/testdata/validation_constants.golden create mode 100644 src/test/e2e/standard/testdata/validation_non_sensitive.golden create mode 100644 src/test/e2e/standard/testdata/validation_with_env_vars.golden create mode 100644 src/test/e2e/standard/testdata/validation_with_set.golden delete mode 100644 src/test/e2e/standard/validation.tpl create mode 100644 src/test/unit/common/validation/validation.bad.tmpl.yaml create mode 100644 src/test/unit/common/validation/validation.tmpl.yaml diff --git a/design-docs/senstive-config.md b/design-docs/senstive-config.md deleted file mode 100644 index 2fdbc70c..00000000 --- a/design-docs/senstive-config.md +++ /dev/null @@ -1,89 +0,0 @@ -# Designing Sensitive Configuration - -Author(s): @meganwolf0 -Date Created: Sept 17, 2024 -Status: DRAFT -Ticket: [#641](https://github.com/defenseunicorns/lula/issues/641) -Reviews Requested By: TBD - -## Problem Statement - -Given that validation data may incorporate sensitive information (e.g., API Keys, passwords, etc.), we need to determine the solution for -1. Mapping those values to their source (e.g., environment variables, config files, secret store, etc.) - -> I think as an initial cut, this should be scoped to variables only coming from the environment, maybe include functionality to source .env files -2. Ensuring that the values are masked in the output of the validation (e.g., when printing the component-definition) -3. Having the ability to manually template the values, specifically in the case of testing the validation content (probably in the purview of `lula dev`) - -## Proposal - -Comparing a few different options: - -### Option 1: Go templates + string replacement -Basically, establish a prefix for the variables that are sensitive, then perform a string repace for the templated values -> also can apply a masking function when running the output? - -(see https://github.com/defenseunicorns/lula/tree/517-go-template-testing for an example of templating something with "secret.xx") - -### Option 2: HCL variable templating -I think this is probably generally more of a heavy-lift to get set-up, and will also introduce more verbosity when setting up variables, but could be a more flexible solution. - -This is actually probably more relevant with generally templating, as it handles complex data structures... - -## Scope and Requirements - -IMO, the best way to define the scope is to determine which examples we are trying to cover: - -1. Templating a sensitive API Key in a notional Domain API-Spec - -```yaml -# ... rest of the validation ... -domain: - type: api - api-spec: - requests: - - name: local - url: https://some.url/v1/api - content-type: application/json - headers: - Authorization: Bearer {{ .secret.api_key }} -# ... rest of the validation ... -``` - -2. Templating a sensitive policy value...? - -```yaml -# ... -# ... -``` - -## Implementation Details - -Tiers of templating: - -* Lula Tools Template -> Default templates constants + variables - -Expand upon the implementation details here. Draw the reader’s attention to: - -* Changes to existing systems. -* Creation of new systems. -* Impacts to the customer. -* Include code samples where possible. - -## Metrics & Alerts - -List any Metrics / Alerts that you plan to include in the system design - -## Alternatives Considered - -List any alternative solutions considered and how the proposed solution is a better fit. - -## Non-Goals - -List out anything that may be related to the solution, but won’t be covered by this solution. - -## Future Improvements: - -List out anything that won’t be included in this version of the feature/solution, but could be revisited or iterated upon in the future. - -## Other Considerations: - -List anything else that won’t be solved by the solution \ No newline at end of file diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index ee0fb2f0..ea9fdbdf 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -37,4 +37,46 @@ lula-config.yaml log_level: debug target: il4 summary: true +``` + +### Templating Configuration Fields + +TODO - description of templating configuration fields + +```yaml +# constants = place to define non-changing values that can be of any type +# I think stuff here probably shouldn't be set by env vars - it's hard to be deterministic because of the character set differences, also type differences could lead to weird side effects +# Another note about this - we could probably easily pull in values of child components if this was referenced from a system-level - so this kind of behaves a bit like help values.yaml +constants: + # map[string]interface{} - elements referenced via template as {{ .const.key }} + type: software + title: lula + # Sample: Istio-specific values + istio: + namespace: istio-system # overriden by --set const.istio.namespace=my-istio-namespace + resources: + jsoncm: configmaps # (NOT) overriden by LULA_VAR_RESOURCES_JSONCM + # Problem with this is that json-cm and json_cm are different yaml keys, but would possibly reconcile to the same thing... so you're getting some side effects here that aren't great. + yamlcm: configmaps + secret: secrets + pod: pods + boolean: false # (NOT) overriden by LULA_VAR_RESOURCES_BOOLEAN + # ok how does this work when they're different types? an env var will always be a string... + exemptions: + - one + - two + - three + +# variables = place to define changing values of string type, and optionally sensitive values +# NOTE - if a variable is defined here, but does not have a default, you will need to make sure it's set either via --set or LULA_VAR_* for the template to execute without error (actually it doesn't error, just prints debug statements) +variables: + - key: some_lula_secret # set by LULA_VAR_SOME_LULA_SECRET / overriden by --set var.some_lula_secret=my-secret + default: blahblah # optional + sensitive: true # {{ var.some_lula_secret | mask }} + - key: some_env_var + default: this-should-be-overridden + +# Lula config values, still accessible via LULA_*, where * is the key +log_level: info +target: il5 ``` \ No newline at end of file diff --git a/lula-config.yaml b/lula-config.yaml index e1c84b3b..da38ff76 100644 --- a/lula-config.yaml +++ b/lula-config.yaml @@ -1,35 +1 @@ - -# constants = place to define non-changing values that can be of any type -# I think stuff here probably shouldn't be set by env vars - it's hard to be deterministic because of the character set differences, also type differences could lead to weird side effects -# Another note about this - we could probably easily pull in values of child components if this was referenced from a system-level - so this kind of behaves a bit like help values.yaml -constants: - # map[string]interface{} - elements referenced via template as {{ .const.key }} - type: software - title: lula - # Sample: Istio-specific values - istio: - namespace: istio-system # overriden by --set const.istio.namespace=my-istio-namespace - resources: - jsoncm: configmaps # (NOT) overriden by LULA_VAR_RESOURCES_JSONCM - # Problem with this is that json-cm and json_cm are different yaml keys, but would possibly reconcile to the same thing... so you're getting some side effects here that aren't great. - yamlcm: configmaps - secret: secrets - pod: pods - boolean: false # (NOT) overriden by LULA_VAR_RESOURCES_BOOLEAN - # ok how does this work when they're different types? an env var will always be a string... - exemptions: - - one - - two - - three - -# variables = place to define changing values of string type, and optionally sensitive values -variables: - - key: some_lula_secret # set by LULA_VAR_SOME_LULA_SECRET / overriden by --set var.some_lula_secret=my-secret - default: blahblah # optional - sensitive: true # {{ var.some_lula_secret | mask }} - - key: some_env_var - default: this-should-be-overridden - -# Lula config values, still accessible via LULA_*, where * is the key -log_level: info -target: il5 +log_level: info \ No newline at end of file diff --git a/src/cmd/tools/template.go b/src/cmd/tools/template.go index e38114d1..c4b47c37 100644 --- a/src/cmd/tools/template.go +++ b/src/cmd/tools/template.go @@ -13,21 +13,8 @@ import ( "github.com/spf13/cobra" ) -type templateFlags struct { - InputFile string // -f --input-file - OutputFile string // -o --output-file - Set []string // --set - All bool // --all - // some demo flags - Const bool // --const - NonSensitive bool // --non-sensitive - Sensitive bool // --sensitive -} - -var templateOpts = &templateFlags{} - var templateHelp = ` -To template an OSCAL Model: +To template an OSCAL Model, defaults to masking sensitive variables: lula tools template -f ./oscal-component.yaml To indicate a specific output file: @@ -37,156 +24,133 @@ To perform overrides on the template data: lula tools template -f ./oscal-component.yaml --set var.key1=value1 --set const.key2=value2 To perform the full template operation, including sensitive data: - lula tools template -f ./oscal-component.yaml --all + lula tools template -f ./oscal-component.yaml --render all Data for templating should be stored under 'constants' or 'variables' configuration items in a lula-config.yaml file See documentation for more detail on configuration schema ` -var templateCmd = &cobra.Command{ - Use: "template", - Short: "Template an artifact", - Long: "Resolving templated artifacts with configuration data", - Args: cobra.NoArgs, - Example: templateHelp, - Run: func(cmd *cobra.Command, args []string) { - // Read file - data, err := pkgCommon.ReadFileToBytes(templateOpts.InputFile) - if err != nil { - message.Fatal(err, err.Error()) - } - - // Get current viper pointer - v := common.GetViper() - - // Get constants and variables for templating from viper config - var constants map[string]interface{} - var variables []template.VariableConfig - - err = v.UnmarshalKey(common.VConstants, &constants) - if err != nil { - message.Fatalf(err, "unable to unmarshal constants into map: %v", err) - } - - err = v.UnmarshalKey(common.VVariables, &variables) - if err != nil { - message.Fatalf(err, "unable to unmarshal variables into slice: %v", err) - } - // Handles merging viper config file data + environment variables - templateData := template.CollectTemplatingData(constants, variables) - - // override anything that's --set - overrideTemplateValues(templateData, templateOpts.Set) - - // Execute the template function based on the flags (TESTING ONLY) - var templatedData []byte - if templateOpts.Const { - templatedData, err = template.ExecuteConstTemplate(templateData.Constants, string(data)) - if err != nil { - message.Fatalf(err, "error templating validation: %v", err) - } - } else if templateOpts.NonSensitive { - templatedData, err = template.ExecuteNonSensitiveTemplate(templateData, string(data)) +func TemplateCommand() *cobra.Command { + var ( + inputFile string + outputFile string + setOpts []string + renderTypeString string + ) + + cmd := &cobra.Command{ + Use: "template", + Short: "Template an artifact", + Long: "Resolving templated artifacts with configuration data", + Args: cobra.NoArgs, + Example: templateHelp, + RunE: func(cmd *cobra.Command, args []string) error { + // Get current viper pointer + v := common.GetViper() + + // Read file + data, err := pkgCommon.ReadFileToBytes(inputFile) if err != nil { - message.Fatalf(err, "error templating validation: %v", err) + return fmt.Errorf("error reading file: %v", err) } - } else if templateOpts.Sensitive { - templatedData, err = template.ExecuteSensitiveTemplate(templateData, string(data)) + + // Validate render type + renderType, err := getRenderType(renderTypeString) if err != nil { - message.Fatalf(err, "error templating validation: %v", err) + message.Warn("invalid render type, defaulting to masked") } - } else if templateOpts.All { - templatedData, err = template.ExecuteFullTemplate(templateData, string(data)) + + // Get constants and variables for templating from viper config + var constants map[string]interface{} + var variables []template.VariableConfig + + err = v.UnmarshalKey(common.VConstants, &constants) if err != nil { - message.Fatalf(err, "error templating validation: %v", err) + return fmt.Errorf("unable to unmarshal constants into map: %v", err) } - } else { - templatedData, err = template.ExecuteMaskedTemplate(templateData, string(data)) + + err = v.UnmarshalKey(common.VVariables, &variables) if err != nil { - message.Fatalf(err, "error templating validation: %v", err) + return fmt.Errorf("unable to unmarshal variables into slice: %v", err) } - } - if templateOpts.OutputFile == "" { - _, err := os.Stdout.Write(templatedData) + // Get overrides from --set flag + overrides := getOverrides(setOpts) + + // Handles merging viper config file data + environment variables + // Throws an error if config keys are invalid for templating + templateData, err := template.CollectTemplatingData(constants, variables, overrides) if err != nil { - message.Fatalf(err, "failed to write to stdout: %v", err) + return fmt.Errorf("error collecting templating data: %v", err) } - } else { - err = files.CreateFileDirs(templateOpts.OutputFile) + + templateRenderer := template.NewTemplateRenderer(string(data), renderType, templateData) + output, err := templateRenderer.Render() if err != nil { - message.Fatalf(err, "failed to create output file path: %s\n", err) + return fmt.Errorf("error rendering template: %v", err) } - err = os.WriteFile(templateOpts.OutputFile, templatedData, 0644) - if err != nil { - message.Fatal(err, err.Error()) + + if outputFile == "" { + _, err := cmd.OutOrStdout().Write(output) + if err != nil { + return fmt.Errorf("failed to write to stdout: %v", err) + } + } else { + err = files.CreateFileDirs(outputFile) + if err != nil { + return fmt.Errorf("failed to create output file path: %v", err) + } + err = os.WriteFile(outputFile, output, 0644) + if err != nil { + return err + } } - } + return nil + }, + } - }, -} + cmd.Flags().StringVarP(&inputFile, "input-file", "f", "", "the path to the target artifact") + cmd.MarkFlagRequired("input-file") + cmd.Flags().StringVarP(&outputFile, "output-file", "o", "", "the path to the output file. If not specified, the output file will be directed to stdout") + cmd.Flags().StringSliceVarP(&setOpts, "set", "s", []string{}, "set a value in the template data") + cmd.Flags().StringVarP(&renderTypeString, "render", "r", "masked", "values to render the template with, options are: masked, constants, non-sensitive, all") -func TemplateCommand() *cobra.Command { - return templateCmd + return cmd } func init() { common.InitViper() + toolsCmd.AddCommand(TemplateCommand()) +} - toolsCmd.AddCommand(templateCmd) - - templateCmd.Flags().StringVarP(&templateOpts.InputFile, "input-file", "f", "", "the path to the target artifact") - templateCmd.MarkFlagRequired("input-file") - templateCmd.Flags().StringVarP(&templateOpts.OutputFile, "output-file", "o", "", "the path to the output file. If not specified, the output file will be directed to stdout") - templateCmd.Flags().StringSliceVarP(&templateOpts.Set, "set", "s", []string{}, "set a value in the template data") - - templateCmd.Flags().BoolVar(&templateOpts.Const, "const", false, "only include constants in the template") - templateCmd.Flags().BoolVar(&templateOpts.NonSensitive, "non-sensitive", false, "only include non-sensitive variables in the template") - templateCmd.Flags().BoolVar(&templateOpts.Sensitive, "sensitive", false, "only include sensitive variables in the template") - templateCmd.Flags().BoolVar(&templateOpts.All, "all", false, "include all variables in the template") +func getRenderType(item string) (template.RenderType, error) { + switch strings.ToLower(item) { + case "masked": + return template.MASKED, nil + case "constants": + return template.CONSTANTS, nil + case "non-sensitive": + return template.NONSENSITIVE, nil + case "all": + return template.ALL, nil + } + return template.MASKED, fmt.Errorf("invalid render type: %s", item) } -func overrideTemplateValues(templateData *template.TemplateData, setFlags []string) { +func getOverrides(setFlags []string) map[string]string { + overrides := make(map[string]string) for _, flag := range setFlags { parts := strings.SplitN(flag, "=", 2) if len(parts) != 2 { - message.Fatalf(fmt.Errorf("invalid --set flag format, should be key.path=value"), "invalid --set flag format, should be key.path=value") + message.Fatalf(fmt.Errorf("invalid --set flag format, should be .root.key=value"), "invalid --set flag format, should be .root.key=value") } - path, value := parts[0], parts[1] - // for each set flag, check if .var or .const - // if .var, set the value in the templateData.Variables - // if .const, set the value in the templateData.Constants - if strings.HasPrefix(path, "."+template.VAR+".") { - // Set the value in the templateData.Variables - key := strings.TrimPrefix(path, "."+template.VAR+".") - templateData.Variables[key] = value - } else if strings.HasPrefix(path, "."+template.CONST+".") { - // Set the value in the templateData.Constants - key := strings.TrimPrefix(path, "."+template.CONST+".") - setNestedValue(templateData.Constants, key, value) + if !strings.HasPrefix(parts[0], "."+template.CONST+".") && !strings.HasPrefix(parts[0], "."+template.VAR+".") { + message.Fatalf(fmt.Errorf("invalid --set flag format, path should start with .const or .var"), "invalid --set flag format, path should start with .const or .var") } - } -} -// Helper function to set a value in a map based on a JSON-like key path -func setNestedValue(m map[string]interface{}, path string, value interface{}) error { - keys := strings.Split(path, ".") - lastKey := keys[len(keys)-1] - - // Traverse the map, creating intermediate maps if necessary - for _, key := range keys[:len(keys)-1] { - if _, exists := m[key]; !exists { - m[key] = make(map[string]interface{}) - } - if nestedMap, ok := m[key].(map[string]interface{}); ok { - m = nestedMap - } else { - return fmt.Errorf("path %s contains a non-map value", key) - } + path, value := parts[0], parts[1] + overrides[path] = value } - - // Set the final value - m[lastKey] = value - return nil + return overrides } diff --git a/src/internal/template/template.go b/src/internal/template/template.go index 0fe60989..a9ff32a4 100644 --- a/src/internal/template/template.go +++ b/src/internal/template/template.go @@ -1,10 +1,13 @@ package template import ( + "fmt" "os" "regexp" "strings" "text/template" + + "github.com/defenseunicorns/lula/src/pkg/message" ) const ( @@ -13,36 +16,82 @@ const ( VAR = "var" ) -func createTemplate() *template.Template { - // Register custom template functions - funcMap := template.FuncMap{ - "concatToRegoList": func(a []any) string { - return concatToRegoList(a) - }, - "mask": func(a string) string { - return "********" - }, - // Add more custom functions as needed +type RenderType string + +const ( + MASKED RenderType = "masked" + CONSTANTS RenderType = "constants" + NONSENSITIVE RenderType = "non-sensitive" + ALL RenderType = "all" +) + +type TemplateRenderer struct { + tpl *template.Template + TemplateString string + *TemplateData + RenderType +} + +func NewTemplateRenderer(templateString string, renderType RenderType, templateData *TemplateData) *TemplateRenderer { + return &TemplateRenderer{ + tpl: createTemplate(), + TemplateString: templateString, + RenderType: renderType, + TemplateData: templateData, } +} - // Parse the template and apply the function map - tpl := template.New("template").Funcs(funcMap) - tpl.Option("missingkey=zero") - return tpl +func (r *TemplateRenderer) SetRenderType(t RenderType) { + r.RenderType = t +} + +func (r *TemplateRenderer) Render() ([]byte, error) { + switch r.RenderType { + case MASKED: + return r.ExecuteMaskedTemplate() + case CONSTANTS: + return r.ExecuteConstTemplate() + case NONSENSITIVE: + return r.ExecuteNonSensitiveTemplate() + case ALL: + return r.ExecuteFullTemplate() + default: + return r.ExecuteMaskedTemplate() + } +} + +type TemplateData struct { + Constants map[string]interface{} + Variables map[string]string + SensitiveVariables map[string]string +} + +func NewTemplateData() *TemplateData { + return &TemplateData{ + Constants: make(map[string]interface{}), + Variables: make(map[string]string), + SensitiveVariables: make(map[string]string), + } +} + +type VariableConfig struct { + Key string + Default string + Sensitive bool } // ExecuteFullTemplate templates everything -func ExecuteFullTemplate(templateData *TemplateData, templateString string) ([]byte, error) { - tpl := createTemplate() - tpl, err := tpl.Parse(templateString) +func (r *TemplateRenderer) ExecuteFullTemplate() ([]byte, error) { + templateString := r.TemplateString + tpl, err := r.tpl.Parse(templateString) if err != nil { return []byte{}, err } var buffer strings.Builder - allVars := MergeStringMaps(templateData.Variables, templateData.SensitiveVariables) + allVars := concatStringMaps(r.TemplateData.Variables, r.TemplateData.SensitiveVariables) err = tpl.Execute(&buffer, map[string]interface{}{ - CONST: templateData.Constants, + CONST: r.TemplateData.Constants, VAR: allVars}) if err != nil { return []byte{}, err @@ -53,20 +102,19 @@ func ExecuteFullTemplate(templateData *TemplateData, templateString string) ([]b // ExecuteConstTemplate templates only constants // this templates only values in the constants map -func ExecuteConstTemplate(constants map[string]interface{}, templateString string) ([]byte, error) { +func (r *TemplateRenderer) ExecuteConstTemplate() ([]byte, error) { // Find anything {{ var.KEY }} and replace with {{ "{{ var.KEY }}" }} re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) - templateString = re.ReplaceAllString(templateString, "{{ \"{{ ."+VAR+".$1 }}\" }}") + templateString := re.ReplaceAllString(r.TemplateString, "{{ \"{{ ."+VAR+".$1 }}\" }}") - tpl := createTemplate() - tpl, err := tpl.Parse(templateString) + tpl, err := r.tpl.Parse(templateString) if err != nil { return []byte{}, err } var buffer strings.Builder err = tpl.Execute(&buffer, map[string]interface{}{ - CONST: constants}) + CONST: r.TemplateData.Constants}) if err != nil { return []byte{}, err } @@ -76,46 +124,27 @@ func ExecuteConstTemplate(constants map[string]interface{}, templateString strin // ExecuteNonSensitiveTemplate templates only constants and non-sensitive variables // used for compose operations -func ExecuteNonSensitiveTemplate(templateData *TemplateData, templateString string) ([]byte, error) { +func (r *TemplateRenderer) ExecuteNonSensitiveTemplate() ([]byte, error) { // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ "{{ var.KEY }}" }} + templateString := r.TemplateString re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) - varMatches := re.FindStringSubmatch(templateString) - for _, m := range varMatches { - if _, ok := templateData.SensitiveVariables[m]; ok { - reSensitive := regexp.MustCompile(`{{\s*\.` + VAR + `\.` + m + `\s*}}`) - templateString = reSensitive.ReplaceAllString(templateString, "{{ \"{{ ."+VAR+"."+m+" }}\" }}") + varMatches := re.FindAllStringSubmatch(templateString, -1) + uniqueMatches := returnUniqueMatches(varMatches, 1) + for k, matches := range uniqueMatches { + if _, ok := r.TemplateData.SensitiveVariables[matches[0]]; ok { + templateString = strings.ReplaceAll(templateString, k, "{{ \""+k+"\" }}") } } - tpl := createTemplate() - tpl, err := tpl.Parse(templateString) - if err != nil { - return []byte{}, err - } - - var buffer strings.Builder - err = tpl.Execute(&buffer, map[string]interface{}{ - CONST: templateData.Constants, - VAR: templateData.Variables}) - if err != nil { - return []byte{}, err - } - - return []byte(buffer.String()), nil -} - -// ExecuteSensitiveTemplate templates the sensitive variables -// for use immediately before validation, after non-sensitive data is templated, results should not be written -func ExecuteSensitiveTemplate(templateData *TemplateData, templateString string) ([]byte, error) { - tpl := createTemplate() - tpl, err := tpl.Parse(templateString) + tpl, err := r.tpl.Parse(templateString) if err != nil { return []byte{}, err } var buffer strings.Builder err = tpl.Execute(&buffer, map[string]interface{}{ - VAR: templateData.SensitiveVariables}) + CONST: r.TemplateData.Constants, + VAR: r.TemplateData.Variables}) if err != nil { return []byte{}, err } @@ -125,28 +154,27 @@ func ExecuteSensitiveTemplate(templateData *TemplateData, templateString string) // ExecuteMaskedTemplate templates all values, but masks the sensitive ones // for display/printing only -func ExecuteMaskedTemplate(templateData *TemplateData, templateString string) ([]byte, error) { +func (r *TemplateRenderer) ExecuteMaskedTemplate() ([]byte, error) { // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ var.KEY | mask }} + templateString := r.TemplateString re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) - varMatches := re.FindStringSubmatch(templateString) - for _, m := range varMatches { - if _, ok := templateData.SensitiveVariables[m]; ok { - reSensitive := regexp.MustCompile(`{{\s*\.` + VAR + `\.` + m + `\s*}}`) - templateString = reSensitive.ReplaceAllString(templateString, "{{ ."+VAR+"."+m+" | mask }}") + varMatches := re.FindAllStringSubmatch(templateString, -1) + uniqueMatches := returnUniqueMatches(varMatches, 1) + for k, matches := range uniqueMatches { + if _, ok := r.TemplateData.SensitiveVariables[matches[0]]; ok { + templateString = strings.ReplaceAll(templateString, k, "********") } } - tpl := createTemplate() - tpl, err := tpl.Parse(templateString) + tpl, err := r.tpl.Parse(templateString) if err != nil { return []byte{}, err } var buffer strings.Builder - allVars := MergeStringMaps(templateData.Variables, templateData.SensitiveVariables) err = tpl.Execute(&buffer, map[string]interface{}{ - CONST: templateData.Constants, - VAR: allVars}) + CONST: r.TemplateData.Constants, + VAR: r.TemplateData.Variables}) if err != nil { return []byte{}, err } @@ -155,19 +183,22 @@ func ExecuteMaskedTemplate(templateData *TemplateData, templateString string) ([ } // Prepare the templateData object for use in templating -func CollectTemplatingData(constants map[string]interface{}, variables []VariableConfig) *TemplateData { +func CollectTemplatingData(constants map[string]interface{}, variables []VariableConfig, overrides map[string]string) (*TemplateData, error) { // Create the TemplateData object from the constants and variables templateData := NewTemplateData() + + // check for invalid characters in keys + err := checkForInvalidKeys(constants, variables) + if err != nil { + return templateData, err + } + templateData.Constants = constants for _, variable := range variables { - // convert '-' to '_' in the key and remove any special characters - variable.Key = strings.ReplaceAll(variable.Key, "-", "_") - re := regexp.MustCompile(`[^a-zA-Z0-9_]`) - variable.Key = re.ReplaceAllString(variable.Key, "") - - templateData.Variables[variable.Key] = variable.Default if variable.Sensitive { templateData.SensitiveVariables[variable.Key] = variable.Default + } else { + templateData.Variables[variable.Key] = variable.Default } } @@ -175,10 +206,27 @@ func CollectTemplatingData(constants map[string]interface{}, variables []Variabl envMap := GetEnvVars(PREFIX) // Update the templateData with the environment variables overrides - templateData.Variables = MergeStringMaps(templateData.Variables, envMap) - templateData.SensitiveVariables = MergeStringMaps(templateData.SensitiveVariables, envMap) + templateData.Variables = mergeStringMaps(templateData.Variables, envMap) + templateData.SensitiveVariables = mergeStringMaps(templateData.SensitiveVariables, envMap) + + // Apply overrides + overrideTemplateValues(templateData, overrides) - return templateData + // Validate that all env vars have values - currently debug prints missing env vars (do we want to return an error?) + var variablesMissing strings.Builder + for k, v := range templateData.Variables { + if v == "" { + variablesMissing.WriteString(fmt.Sprintf("variable %s is missing a value;\n", k)) + } + } + for k, v := range templateData.SensitiveVariables { + if v == "" { + variablesMissing.WriteString(fmt.Sprintf("sensitive variable %s is missing a value;\n", k)) + } + } + message.Debugf(variablesMissing.String()) + + return templateData, nil } // get all environment variables with the established prefix @@ -210,9 +258,26 @@ func GetEnvVars(prefix string) map[string]string { return envMap } -// MergeStringMaps merges two maps of strings into a single map of strings. -// m2 will overwrite m1 if a key exists in both maps. -func MergeStringMaps(m1, m2 map[string]string) map[string]string { +// createTemplate creates a new template object +func createTemplate() *template.Template { + // Register custom template functions + funcMap := template.FuncMap{ + "concatToRegoList": func(a []any) string { + return concatToRegoList(a) + }, + // Add more custom functions as needed + } + + // Parse the template and apply the function map + tpl := template.New("template").Funcs(funcMap) + tpl.Option("missingkey=error") + + return tpl +} + +// mergeStringMaps merges two maps of strings into a single map of strings. +// m2 will overwrite m1 if a key exists in both maps, similar to left-join operation +func mergeStringMaps(m1, m2 map[string]string) map[string]string { r := map[string]string{} for key, value := range m1 { @@ -220,8 +285,122 @@ func MergeStringMaps(m1, m2 map[string]string) map[string]string { } for key, value := range m2 { + // only add the key if it does exist in r + if _, ok := r[key]; ok { + r[key] = value + } + } + + return r +} + +// concatStringMaps concatenates two maps of strings into a single map of strings. +// m2 will overwrite m1 if a key exists in both maps +func concatStringMaps(m1, m2 map[string]string) map[string]string { + r := make(map[string]string) + + for key, value := range m1 { r[key] = value } + for key, value := range m2 { + r[key] = value + } return r } + +// returnUniqueMatches returns a slice of unique matches from a slice of strings +func returnUniqueMatches(matches [][]string, captures int) map[string][]string { + uniqueMatches := make(map[string][]string) + for _, match := range matches { + fullMatch := match[0] + if _, exists := uniqueMatches[fullMatch]; !exists { + uniqueMatches[fullMatch] = match[captures:] + } + } + return uniqueMatches +} + +// checkForInvalidKeys checks for invalid characters in keys for go text/template +// cannot contain '-' or '.' +func checkForInvalidKeys(constants map[string]interface{}, variables []VariableConfig) error { + var errors strings.Builder + + containsInvalidChars := func(key string) { + if strings.Contains(key, "-") { + errors.WriteString(fmt.Sprintf("invalid key %s - cannot contain '-';", key)) + } + if strings.Contains(key, ".") { + errors.WriteString(fmt.Sprintf("invalid key %s - cannot contain '.';", key)) + } + } + + // check for invalid characters in keys, recursively through constants + var validateKeys func(m map[string]interface{}) + validateKeys = func(m map[string]interface{}) { + for key, value := range m { + containsInvalidChars(key) + if nestedMap, ok := value.(map[string]interface{}); ok { + validateKeys(nestedMap) + } + } + } + + validateKeys(constants) + + // check for invalid characters in keys in variables + for _, variable := range variables { + containsInvalidChars(variable.Key) + } + + if errors.Len() > 0 { + return fmt.Errorf(errors.String()[:len(errors.String())-1]) + } + + return nil +} + +// overrideTemplateValues overrides values in the templateData object with values from the overrides map +func overrideTemplateValues(templateData *TemplateData, overrides map[string]string) { + for path, value := range overrides { + // for each key, check if .var or .const + // if .var, set the value in the templateData.Variables or templateData.SensitiveVariables + // if .const, set the value in the templateData.Constants + if strings.HasPrefix(path, "."+VAR+".") { + key := strings.TrimPrefix(path, "."+VAR+".") + + if _, ok := templateData.SensitiveVariables[key]; ok { + templateData.SensitiveVariables[key] = value + } else { + templateData.Variables[key] = value + } + } else if strings.HasPrefix(path, "."+CONST+".") { + // Set the value in the templateData.Constants + key := strings.TrimPrefix(path, "."+CONST+".") + setNestedValue(templateData.Constants, key, value) + } + } +} + +// Helper function to set a value in a map based on a JSON-like key path +// Only supports basic map path (root.key.subkey) +func setNestedValue(m map[string]interface{}, path string, value interface{}) error { + keys := strings.Split(path, ".") + lastKey := keys[len(keys)-1] + + // Traverse the map, creating intermediate maps if necessary + for _, key := range keys[:len(keys)-1] { + if _, exists := m[key]; !exists { + m[key] = make(map[string]interface{}) + } + if nestedMap, ok := m[key].(map[string]interface{}); ok { + m = nestedMap + } else { + return fmt.Errorf("path %s contains a non-map value", key) + } + } + + // Set the final value + m[lastKey] = value + return nil +} diff --git a/src/internal/template/template_test.go b/src/internal/template/template_test.go index d922fcf2..4a917fbb 100644 --- a/src/internal/template/template_test.go +++ b/src/internal/template/template_test.go @@ -1,6 +1,7 @@ package template_test import ( + "fmt" "os" "reflect" "strings" @@ -9,43 +10,626 @@ import ( "github.com/defenseunicorns/lula/src/internal/template" ) -// func TestExecuteTemplate(t *testing.T) { +func testRender(t *testing.T, templateRenderer *template.TemplateRenderer, expected string) error { + t.Helper() -// test := func(t *testing.T, data map[string]interface{}, preTemplate string, expected string) { -// t.Helper() -// // templateData returned -// got, err := template.ExecuteTemplate(data, preTemplate) -// if err != nil { -// t.Fatalf("error templating data: %s\n", err.Error()) -// } + got, err := templateRenderer.Render() + if err != nil { + return fmt.Errorf("error templating data: %v\n", err.Error()) + } + + if string(got) != expected { + t.Fatalf("Expected %s - Got %s\n", expected, string(got)) + } + return nil +} + +func TestExecuteFullTemplate(t *testing.T) { + t.Parallel() + + t.Run("Test template all with data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + secret template: my-secret + ` + + tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) + err := testRender(t, tr, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template all with all empty data, error", func(t *testing.T) { + templateData := template.NewTemplateData() + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) + err := testRender(t, tr, "") + if err == nil { + t.Fatalf("Expected an error, but got nil") + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template all with invalid template paths, error", func(t *testing.T) { + templateData := template.NewTemplateData() + + templateString := ` + constant template: {{ .constant.testVar }} + ` + + tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) + err := testRender(t, tr, "") + if err == nil { + t.Fatal("expected error, got nil") + } + }) -// if string(got) != expected { -// t.Fatalf("Expected %s - Got %s\n", expected, string(got)) -// } -// } + t.Run("Test template all with invalid template characters, error", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "test-var": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } -// t.Run("Test {{ .testVar }} with data", func(t *testing.T) { -// data := map[string]interface{}{ -// "testVar": "testing", -// } + templateString := ` + constant template: {{ .const.test-var }} + ` -// test(t, data, "{{ .testVar }}", "testing") -// }) + tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) + err := testRender(t, tr, "") + if err == nil { + t.Fatal("expected error, got nil") + } + }) -// t.Run("Test {{ .testVar }} but empty data", func(t *testing.T) { -// data := map[string]interface{}{} + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template all with invalid template subpath, error", func(t *testing.T) { + templateData := template.NewTemplateData() + + templateString := ` + variable template: {{ .var.nokey.sub }} + ` + + tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) + err := testRender(t, tr, "") + if err == nil { + t.Fatal("expected error, got nil") + } + }) -// test(t, data, "{{ .testVar }}", "") -// }) + t.Run("Test template all with invalid template, error", func(t *testing.T) { + templateData := template.NewTemplateData() + + templateString := ` + constant template: {{ constant.testVar }} + ` + tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) + err := testRender(t, tr, "") + if err == nil { + t.Fatal("expected error, got nil") + } + }) + + t.Run("Test template on 'concatToRegoList' function", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "exemptions": []interface{}{"one", "two", "three"}, + }, + } + + templateString := ` + constant template: {{ .const.exemptions | concatToRegoList }} + ` + expected := ` + constant template: "one", "two", "three" + ` + tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) + err := testRender(t, tr, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) -// } +} + +func TestExecuteConstTemplate(t *testing.T) { + t.Parallel() + + t.Run("Test template const with data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateString, template.CONSTANTS, templateData) + err := testRender(t, tr, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test template const with missing var data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + tr := template.NewTemplateRenderer(templateString, template.CONSTANTS, templateData) + err := testRender(t, tr, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test template const with weird spacing in template", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{.var.some_env_var}} + secret template: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + tr := template.NewTemplateRenderer(templateString, template.CONSTANTS, templateData) + err := testRender(t, tr, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template const with empty data", func(t *testing.T) { + templateData := &template.TemplateData{ + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateString, template.CONSTANTS, templateData) + err := testRender(t, tr, "") + if err == nil { + t.Fatalf("Expected an error, but got nil") + } + }) + +} + +func TestExecuteNonSensitiveTemplate(t *testing.T) { + t.Parallel() + + t.Run("Test template nonsensitive with data and duplicate matches", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + variable template2: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + secret template2: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + variable template2: my-env-var + secret template: {{ .var.some_lula_secret }} + secret template2: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateString, template.NONSENSITIVE, templateData) + err := testRender(t, tr, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test template nonsensitive with weird spacing in template", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{.var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + secret template: {{.var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateString, template.NONSENSITIVE, templateData) + err := testRender(t, tr, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template nonsensitive with empty var data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateString, template.NONSENSITIVE, templateData) + err := testRender(t, tr, "") + if err == nil { + t.Fatalf("Expected an error, but got nil") + } + }) + +} + +func TestExecuteMaskedTemplate(t *testing.T) { + t.Parallel() + + t.Run("Test masked template with sensitive data and duplicates", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + variable template2: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + secret template2: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + variable template2: my-env-var + secret template: ******** + secret template2: ******** + ` + + tr := template.NewTemplateRenderer(templateString, template.MASKED, templateData) + err := testRender(t, tr, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test masked template with weird spacing in template", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{.var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + secret template: ******** + ` + + tr := template.NewTemplateRenderer(templateString, template.MASKED, templateData) + err := testRender(t, tr, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test masked template with missing data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateString, template.MASKED, templateData) + err := testRender(t, tr, "") + if err == nil { + t.Fatalf("Expected an error, but got nil") + } + }) +} + +func TestCollectTemplatingData(t *testing.T) { + + test := func(t *testing.T, constants map[string]interface{}, variables []template.VariableConfig, overrides map[string]string, expected *template.TemplateData) error { + t.Helper() + // templateData returned + got, err := template.CollectTemplatingData(constants, variables, overrides) + if err != nil { + return err + } + + if !reflect.DeepEqual(got, expected) { + t.Fatalf("Expected %v - Got %v\n", expected, got) + } + return nil + } + + t.Run("Test collect templating data", func(t *testing.T) { + var overrides map[string]string + constants := map[string]interface{}{ + "testVar": "testing", + } + variables := []template.VariableConfig{ + { + Key: "some_env_var", + Default: "my-env-var", + Sensitive: false, + }, + { + Key: "some_lula_secret", + Default: "my-secret", + Sensitive: true, + }, + } + expected := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + err := test(t, constants, variables, overrides, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test collect templating data with env vars", func(t *testing.T) { + os.Setenv("LULA_VAR_SOME_LULA_SECRET", "env-secret") + os.Setenv("LULA_VAR_SOME_ENV_VAR", "env-var") + defer os.Unsetenv("LULA_VAR_SOME_LULA_SECRET") + defer os.Unsetenv("LULA_VAR_SOME_ENV_VAR") + + var overrides map[string]string + constants := map[string]interface{}{ + "testVar": "testing", + } + variables := []template.VariableConfig{ + { + Key: "some_env_var", + Default: "my-env-var", + Sensitive: false, + }, + { + Key: "some_lula_secret", + Default: "my-secret", + Sensitive: true, + }, + } + expected := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "env-secret", + }, + } + err := test(t, constants, variables, overrides, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test collect templating data with overrides", func(t *testing.T) { + overrides := map[string]string{ + ".var.some_env_var": "override-var", + ".var.some_lula_secret": "override-secret", + ".const.test.subkey": "override-subkey", + } + constants := map[string]interface{}{ + "testVar": "testing", + "test": map[string]interface{}{ + "subkey": "subkey-value", + }, + } + variables := []template.VariableConfig{ + { + Key: "some_env_var", + Default: "my-env-var", + Sensitive: false, + }, + { + Key: "some_lula_secret", + Default: "my-secret", + Sensitive: true, + }, + } + expected := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + "test": map[string]interface{}{ + "subkey": "override-subkey", + }, + }, + Variables: map[string]string{ + "some_env_var": "override-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "override-secret", + }, + } + err := test(t, constants, variables, overrides, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test collect templating data with bad keys", func(t *testing.T) { + var overrides map[string]string + constants := map[string]interface{}{ + "test-var": "testing", + "anotherkey": map[string]interface{}{ + "sub.key": "testing", + }, + } + variables := []template.VariableConfig{ + { + Key: "some-env-var", + Default: "my-env-var", + Sensitive: false, + }, + } + expected := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{}, + } + err := test(t, constants, variables, overrides, expected) + if err == nil { + t.Fatal("expected error, got nil") + } + numErrors := strings.Count(err.Error(), "invalid key") + if numErrors != 3 { + t.Fatalf("expected 3 invalid constant keys, got %d", numErrors) + } + }) +} func TestGetEnvVars(t *testing.T) { test := func(t *testing.T, prefix string, key string, value string) { t.Helper() - // os.Setenv(key, value) + os.Setenv(key, value) + defer os.Unsetenv(key) envMap := template.GetEnvVars(prefix) // convert key to expected format @@ -54,7 +638,6 @@ func TestGetEnvVars(t *testing.T) { if envMap[strings.ToLower(strippedKey)] != value { t.Fatalf("Expected %s - Got %s\n", value, envMap[strings.ToLower(strippedKey)]) } - os.Unsetenv(key) } t.Run("Test LULA_RESOURCE - Pass", func(t *testing.T) { @@ -65,27 +648,3 @@ func TestGetEnvVars(t *testing.T) { test(t, "OTHER_", "OTHER_RESOURCE", "deployments") }) } - -func TestMergeStringMaps(t *testing.T) { - t.Parallel() // Enable parallel execution of tests - t.Run("Should merge two maps", func(t *testing.T) { - t.Parallel() // Enable parallel execution of subtests - m1 := map[string]string{ - "key1": "value1", - "key2": "value2", - } - m2 := map[string]string{ - "key1": "value3", - "key3": "value4", - } - expected := map[string]string{ - "key1": "value3", - "key2": "value2", - "key3": "value4", - } - result := template.MergeStringMaps(m1, m2) - if !reflect.DeepEqual(result, expected) { - t.Errorf("MergeStringMaps() got = %v, want %v", result, expected) - } - }) -} diff --git a/src/internal/template/types.go b/src/internal/template/types.go deleted file mode 100644 index 84dd4d78..00000000 --- a/src/internal/template/types.go +++ /dev/null @@ -1,21 +0,0 @@ -package template - -type VariableConfig struct { - Key string - Default string - Sensitive bool -} - -type TemplateData struct { - Constants map[string]interface{} - Variables map[string]string - SensitiveVariables map[string]string -} - -func NewTemplateData() *TemplateData { - return &TemplateData{ - Constants: make(map[string]interface{}), - Variables: make(map[string]string), - SensitiveVariables: make(map[string]string), - } -} diff --git a/src/test/e2e/standard/lula-config.yaml b/src/test/e2e/standard/lula-config.yaml index 92a84fdb..911f39ff 100644 --- a/src/test/e2e/standard/lula-config.yaml +++ b/src/test/e2e/standard/lula-config.yaml @@ -1,33 +1,20 @@ - -# If we had remote variables, e.g., co-located with the oscal data, are those automatically pulled in? and what is the override mechanism? Probably the system overrides the components... really this is just like re-hashing helm charts with values.yaml... -# If we wanted to move the validations to be CRDs at some point, this could give us a leg-up... - -# I think stuff here maybe shouldn't be set by env vars - as described it's hard to be deterministic constants: -# since this field can take anything, even list data, there's not really a good mechanism for validating the data type: software title: lula - # Example: Istio-specific values - istio: - namespace: istio-system # overriden by --set const.istio.namespace=my-istio-namespace + resources: - jsoncm: configmaps # overriden by LULA_VAR_RESOURCES_JSONCM - # Problem with this is that json-cm and json_cm are different yaml keys, but would possibly reconcile to the same thing... so you're getting some side effects here that aren't great. - yamlcm: configmaps - secret: secrets - pod: pods - boolean: false # overriden by LULA_VAR_RESOURCES_BOOLEAN - # ok how does this work when they're different types? an env var will always be a string... -# how to handle "set" + name: test-pod-label + namespace: validation-test + exemptions: + - one + - two + - three -# string only variables, basically just from env vars -# the way zarf does this is they have to be defined here to be available, I believe variables: - - key: some_lula_secret # LULA_VAR_SOME_LULA_SECRET? # overriden by --set var.some_lula_secret=my-secret - default: blahblah # optional - sensitive: true # {{ var.some_lula_secret }} -> look-up + - key: some_lula_secret + sensitive: true - key: some_env_var + default: this-should-be-overridden -# Lula config values log_level: info target: il5 diff --git a/src/test/e2e/standard/template_test.go b/src/test/e2e/standard/template_test.go index 4fc25786..d2f84528 100644 --- a/src/test/e2e/standard/template_test.go +++ b/src/test/e2e/standard/template_test.go @@ -1,19 +1,21 @@ package test import ( - "bytes" + "flag" "os" + "path/filepath" - "strings" "testing" "github.com/defenseunicorns/lula/src/cmd" "github.com/defenseunicorns/lula/src/test/util" ) +var updateGolden = flag.Bool("update", false, "update golden files") + func TestTemplateCommand(t *testing.T) { - test := func(t *testing.T, expectError bool, args ...string) (string, error) { + test := func(t *testing.T, goldenFileName string, expectError bool, args ...string) error { t.Helper() cmdArgs := []string{"tools", "template"} @@ -22,60 +24,101 @@ func TestTemplateCommand(t *testing.T) { cmd := cmd.RootCommand() _, output, err := util.ExecuteCommand(cmd, cmdArgs...) - if err != nil && !expectError { - t.Fatal(err) + if err != nil { + if !expectError { + return err + } else { + return nil + } } - return output, err + if !expectError { + goldenFile := filepath.Join("testdata", goldenFileName+".golden") + + if *updateGolden && !expectError { + err = os.WriteFile(goldenFile, []byte(output), 0644) + if err != nil { + return err + } + } + + expected, err := os.ReadFile(goldenFile) + if err != nil { + return err + } + + if output != string(expected) { + t.Fatalf("Expected:\n%s\n - Got \n%s\n", expected, output) + } + } + + return nil } - t.Run("Template Valid File", func(t *testing.T) { + t.Run("Template Validation", func(t *testing.T) { + err := test(t, "validation", false, "-f", "../../unit/common/validation/validation.tmpl.yaml") + if err != nil { + t.Fatal(err) + } + }) - _, err := test(t, false, "-f", "../../unit/common/oscal/valid-component-template.yaml", "-o", "valid.yaml") - defer os.Remove("valid.yaml") + t.Run("Template Validation with env vars", func(t *testing.T) { + os.Setenv("LULA_VAR_SOME_ENV_VAR", "my-env-var") + defer os.Unsetenv("LULA_VAR_SOME_ENV_VAR") + err := test(t, "validation_with_env_vars", false, "-f", "../../unit/common/validation/validation.tmpl.yaml") if err != nil { t.Fatal(err) } + }) - // this comparison using golden files would make more sense - templated, err := os.ReadFile("valid.yaml") + t.Run("Template Validation with set", func(t *testing.T) { + err := test(t, "validation_with_set", false, "-f", "../../unit/common/validation/validation.tmpl.yaml", "--set", ".const.resources.name=foo") if err != nil { t.Fatal(err) } + }) - valid, err := os.ReadFile("../../unit/common/oscal/valid-component.yaml") + t.Run("Template Validation for all", func(t *testing.T) { + os.Setenv("LULA_VAR_SOME_LULA_SECRET", "env-secret") + defer os.Unsetenv("LULA_VAR_SOME_LULA_SECRET") + err := test(t, "validation_all", false, "-f", "../../unit/common/validation/validation.tmpl.yaml", "--render", "all") if err != nil { t.Fatal(err) } + }) - if !bytes.Equal(templated, valid) { - t.Fatalf("Expected: \n%s\n - Got \n%s\n", valid, templated) + t.Run("Template Validation for non-sensitive", func(t *testing.T) { + err := test(t, "validation_non_sensitive", false, "-f", "../../unit/common/validation/validation.tmpl.yaml", "--render", "non-sensitive") + if err != nil { + t.Fatal(err) } + }) + t.Run("Template Validation for constants", func(t *testing.T) { + err := test(t, "validation_constants", false, "-f", "../../unit/common/validation/validation.tmpl.yaml", "--render", "constants") + if err != nil { + t.Fatal(err) + } }) t.Run("Test help", func(t *testing.T) { - out, _ := test(t, false, "--help") - - if !strings.Contains(out, "Resolving templated artifacts with configuration data") { - t.Fatalf("Expected help string") + err := test(t, "help", false, "--help") + if err != nil { + t.Fatal(err) } }) - // Tests that execute unhappy-paths will hit a fatal message which exits the runtime - // TODO: review RunE command execution and ensure we don't prematurely exit where errors would still be valuable - // t.Run("Test non-existent file", func(t *testing.T) { - // out, _ := test(t, true, "-f", "non-existent.yaml") - - // if !strings.Contains(out, "Path: non-existent.yaml does not exist - unable to digest document") { - // t.Fatalf("Expected error with unable to digest document error") - // } - // }) + t.Run("Template Validation - invalid file error", func(t *testing.T) { + err := test(t, "empty", true, "-f", "not-a-file.yaml") + if err != nil { + t.Fatal(err) + } + }) + t.Run("Template Validation - invalid file schema error", func(t *testing.T) { + err := test(t, "empty", true, "-f", "../../unit/common/validation/validation.bad.tmpl.yaml") + if err != nil { + t.Fatal(err) + } + }) } - -// TODO: -// 1. Test --set for variables -// 2. Test with env vars + config vars -// 3. Test a type mismatch -// 4. Test secrets diff --git a/src/test/e2e/standard/testdata/empty.golden b/src/test/e2e/standard/testdata/empty.golden new file mode 100644 index 00000000..e69de29b diff --git a/src/test/e2e/standard/testdata/help.golden b/src/test/e2e/standard/testdata/help.golden new file mode 100644 index 00000000..535327d8 --- /dev/null +++ b/src/test/e2e/standard/testdata/help.golden @@ -0,0 +1,32 @@ +Resolving templated artifacts with configuration data + +Usage: + lula tools template [flags] + +Examples: + +To template an OSCAL Model, defaults to masking sensitive variables: + lula tools template -f ./oscal-component.yaml + +To indicate a specific output file: + lula tools template -f ./oscal-component.yaml -o templated-oscal-component.yaml + +To perform overrides on the template data: + lula tools template -f ./oscal-component.yaml --set var.key1=value1 --set const.key2=value2 + +To perform the full template operation, including sensitive data: + lula tools template -f ./oscal-component.yaml --render all + +Data for templating should be stored under 'constants' or 'variables' configuration items in a lula-config.yaml file +See documentation for more detail on configuration schema + + +Flags: + -h, --help help for template + -f, --input-file string the path to the target artifact + -o, --output-file string the path to the output file. If not specified, the output file will be directed to stdout + -r, --render string values to render the template with, options are: masked, constants, non-sensitive, all (default "masked") + -s, --set strings set a value in the template data + +Global Flags: + -l, --log-level string Log level when running Lula. Valid options are: warn, info, debug, trace (default "info") diff --git a/src/test/e2e/standard/testdata/validation.golden b/src/test/e2e/standard/testdata/validation.golden new file mode 100644 index 00000000..ff319eb2 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: test-pod-label + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "this-should-be-overridden" == "my-env-var" + "********" == "********" + } + msg = validate.msg + + value_of_my_secret := ******** \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_all.golden b/src/test/e2e/standard/testdata/validation_all.golden new file mode 100644 index 00000000..dc22f17b --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_all.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: foo + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "this-should-be-overridden" == "my-env-var" + "env-secret" == "********" + } + msg = validate.msg + + value_of_my_secret := env-secret \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_constants.golden b/src/test/e2e/standard/testdata/validation_constants.golden new file mode 100644 index 00000000..91e814e7 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_constants.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: foo + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "{{ .var.some_env_var }}" == "my-env-var" + "{{ .var.some_lula_secret }}" == "********" + } + msg = validate.msg + + value_of_my_secret := {{ .var.some_lula_secret }} \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_non_sensitive.golden b/src/test/e2e/standard/testdata/validation_non_sensitive.golden new file mode 100644 index 00000000..e1282c92 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_non_sensitive.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: foo + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "this-should-be-overridden" == "my-env-var" + "{{ .var.some_lula_secret }}" == "********" + } + msg = validate.msg + + value_of_my_secret := {{ .var.some_lula_secret }} \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_with_env_vars.golden b/src/test/e2e/standard/testdata/validation_with_env_vars.golden new file mode 100644 index 00000000..c429cbc2 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_with_env_vars.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: test-pod-label + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "my-env-var" == "my-env-var" + "********" == "********" + } + msg = validate.msg + + value_of_my_secret := ******** \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_with_set.golden b/src/test/e2e/standard/testdata/validation_with_set.golden new file mode 100644 index 00000000..f7be2dc4 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_with_set.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: foo + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "this-should-be-overridden" == "my-env-var" + "********" == "********" + } + msg = validate.msg + + value_of_my_secret := ******** \ No newline at end of file diff --git a/src/test/e2e/standard/validation.tpl b/src/test/e2e/standard/validation.tpl deleted file mode 100644 index ff4a7dc0..00000000 --- a/src/test/e2e/standard/validation.tpl +++ /dev/null @@ -1,30 +0,0 @@ -domain: - type: api - api-spec: - requests: - - name: local - url: https://some.url/v1/api - content-type: application/json - headers: - Authorization: Bearer {{ .var.some_lula_secret }} -provider: - type: opa - opa-spec: - rego: | - package validate - import rego.v1 - - # Default values - default validate := false - default msg := "Not evaluated" - - # Validation result - validate if { - input.jsoncm.name == {{ .const.resources.jsoncm }} - input.yamlcm.logging.level == {{ .const.resources.yamlcm }} - not input.secret.name in { {{ .const.exemptions | concatToRegoList }} } # tmpl creates: "one", "two", "three" - } - msg = validate.msg - - test_env_var := {{ .var.some_env_var }} # non-sensitive - test_another_env_var := {{ .var.another_env_var }} # non-sensitive, no default \ No newline at end of file diff --git a/src/test/unit/common/validation/validation.bad.tmpl.yaml b/src/test/unit/common/validation/validation.bad.tmpl.yaml new file mode 100644 index 00000000..3ea347df --- /dev/null +++ b/src/test/unit/common/validation/validation.bad.tmpl.yaml @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: {{ constant.resources.name }} + version: v1 + resource: pods + namespaces: [{{ .const.missing-key }}] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { {{ .const.resources.exemptions | concatToRegoList }} } + "{{ .var.some_env_var }}" == "my-env-var" + "{{ .var.some_lula_secret }}" == "********" + } + msg = validate.msg + + value_of_my_secret := {{ .var.some_lula_secret }} \ No newline at end of file diff --git a/src/test/unit/common/validation/validation.tmpl.yaml b/src/test/unit/common/validation/validation.tmpl.yaml new file mode 100644 index 00000000..f9eb5baf --- /dev/null +++ b/src/test/unit/common/validation/validation.tmpl.yaml @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: {{ .const.resources.name }} + version: v1 + resource: pods + namespaces: [{{ .const.resources.namespace }}] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { {{ .const.resources.exemptions | concatToRegoList }} } + "{{ .var.some_env_var }}" == "my-env-var" + "{{ .var.some_lula_secret }}" == "********" + } + msg = validate.msg + + value_of_my_secret := {{ .var.some_lula_secret }} \ No newline at end of file diff --git a/src/test/util/utils.go b/src/test/util/utils.go index cb1663d7..978b3990 100644 --- a/src/test/util/utils.go +++ b/src/test/util/utils.go @@ -124,9 +124,12 @@ func ExecuteCommandC(cmd *cobra.Command, args ...string) (c *cobra.Command, outp cmd.SetErr(buf) cmd.SetArgs(args) - cmd.Execute() + execErr := cmd.Execute() - out, err := io.ReadAll(buf) + out, readErr := io.ReadAll(buf) + if readErr != nil { + return cmd, "", readErr + } - return cmd, string(out), err + return cmd, string(out), execErr } From fc5a9cfb1d853fbb68a8db9f666ff5bc9c5d2990 Mon Sep 17 00:00:00 2001 From: Megan Wolf Date: Thu, 19 Sep 2024 21:01:35 -0400 Subject: [PATCH 3/9] fix: go mod tidy --- go.mod | 3 --- go.sum | 8 -------- 2 files changed, 11 deletions(-) diff --git a/go.mod b/go.mod index 0908d249..e63d8f8c 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/charmbracelet/lipgloss v0.13.0 github.com/charmbracelet/x/exp/teatest v0.0.0-20240913162256-9ef7ff40e654 github.com/defenseunicorns/go-oscal v0.6.0 - github.com/defenseunicorns/pkg/helpers v1.1.3 github.com/hashicorp/go-version v1.7.0 github.com/kyverno/kyverno-json v0.0.3 github.com/mattn/go-runewidth v0.0.16 @@ -114,7 +113,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/otiai10/copy v1.14.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect @@ -171,7 +169,6 @@ require ( k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/metrics v0.31.1 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - oras.land/oras-go/v2 v2.5.0 // indirect sigs.k8s.io/controller-runtime v0.18.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.17.2 // indirect diff --git a/go.sum b/go.sum index 7ac6f82a..71bb4a3b 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,6 @@ github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= github.com/defenseunicorns/go-oscal v0.6.0 h1:eflEKfk7edu4L4kWf6aNQpS94ljfGP8lgWpsPYNtE1Q= github.com/defenseunicorns/go-oscal v0.6.0/go.mod h1:UHp2yK9ty2mYJDun7oNhbstCq6SAAwP4YGbw9n7uG6o= -github.com/defenseunicorns/pkg/helpers v1.1.3 h1:EVVuniq02qfAouR//AT0eoCngLWfFORj8H6+pI8M7uo= -github.com/defenseunicorns/pkg/helpers v1.1.3/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= @@ -288,10 +286,6 @@ github.com/open-policy-agent/opa v0.68.0 h1:Jl3U2vXRjwk7JrHmS19U3HZO5qxQRinQbJ2e github.com/open-policy-agent/opa v0.68.0/go.mod h1:5E5SvaPwTpwt2WM177I9Z3eT7qUpmOGjk1ZdHs+TZ4w= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= -github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= -github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= -github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -553,8 +547,6 @@ k8s.io/metrics v0.31.1 h1:h4I4dakgh/zKflWYAOQhwf0EXaqy8LxAIyE/GBvxqRc= k8s.io/metrics v0.31.1/go.mod h1:JuH1S9tJiH9q1VCY0yzSCawi7kzNLsDzlWDJN4xR+iA= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= -oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= sigs.k8s.io/controller-runtime v0.18.2 h1:RqVW6Kpeaji67CY5nPEfRz6ZfFMk0lWQlNrLqlNpx+Q= sigs.k8s.io/controller-runtime v0.18.2/go.mod h1:tuAt1+wbVsXIT8lPtk5RURxqAnq7xkpv2Mhttslg7Hw= sigs.k8s.io/e2e-framework v0.4.0 h1:4yYmFDNNoTnazqmZJXQ6dlQF1vrnDbutmxlyvBpC5rY= From 8c9b12889557cf8bd50567819e04df7f26f8fd1c Mon Sep 17 00:00:00 2001 From: Megan Wolf Date: Fri, 20 Sep 2024 08:19:43 -0400 Subject: [PATCH 4/9] fix: small refactor on template render --- src/cmd/tools/template.go | 4 +- src/internal/template/template.go | 49 ++++++++---------- src/internal/template/template_test.go | 72 +++++++++++++------------- 3 files changed, 59 insertions(+), 66 deletions(-) diff --git a/src/cmd/tools/template.go b/src/cmd/tools/template.go index c4b47c37..55c8b512 100644 --- a/src/cmd/tools/template.go +++ b/src/cmd/tools/template.go @@ -84,8 +84,8 @@ func TemplateCommand() *cobra.Command { return fmt.Errorf("error collecting templating data: %v", err) } - templateRenderer := template.NewTemplateRenderer(string(data), renderType, templateData) - output, err := templateRenderer.Render() + templateRenderer := template.NewTemplateRenderer(string(data), templateData) + output, err := templateRenderer.Render(renderType) if err != nil { return fmt.Errorf("error rendering template: %v", err) } diff --git a/src/internal/template/template.go b/src/internal/template/template.go index a9ff32a4..5bae9f57 100644 --- a/src/internal/template/template.go +++ b/src/internal/template/template.go @@ -27,26 +27,20 @@ const ( type TemplateRenderer struct { tpl *template.Template - TemplateString string - *TemplateData - RenderType + templateString string + templateData *TemplateData } -func NewTemplateRenderer(templateString string, renderType RenderType, templateData *TemplateData) *TemplateRenderer { +func NewTemplateRenderer(templateString string, templateData *TemplateData) *TemplateRenderer { return &TemplateRenderer{ tpl: createTemplate(), - TemplateString: templateString, - RenderType: renderType, - TemplateData: templateData, + templateString: templateString, + templateData: templateData, } } -func (r *TemplateRenderer) SetRenderType(t RenderType) { - r.RenderType = t -} - -func (r *TemplateRenderer) Render() ([]byte, error) { - switch r.RenderType { +func (r *TemplateRenderer) Render(t RenderType) ([]byte, error) { + switch t { case MASKED: return r.ExecuteMaskedTemplate() case CONSTANTS: @@ -82,16 +76,15 @@ type VariableConfig struct { // ExecuteFullTemplate templates everything func (r *TemplateRenderer) ExecuteFullTemplate() ([]byte, error) { - templateString := r.TemplateString - tpl, err := r.tpl.Parse(templateString) + tpl, err := r.tpl.Parse(r.templateString) if err != nil { return []byte{}, err } var buffer strings.Builder - allVars := concatStringMaps(r.TemplateData.Variables, r.TemplateData.SensitiveVariables) + allVars := concatStringMaps(r.templateData.Variables, r.templateData.SensitiveVariables) err = tpl.Execute(&buffer, map[string]interface{}{ - CONST: r.TemplateData.Constants, + CONST: r.templateData.Constants, VAR: allVars}) if err != nil { return []byte{}, err @@ -105,16 +98,16 @@ func (r *TemplateRenderer) ExecuteFullTemplate() ([]byte, error) { func (r *TemplateRenderer) ExecuteConstTemplate() ([]byte, error) { // Find anything {{ var.KEY }} and replace with {{ "{{ var.KEY }}" }} re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) - templateString := re.ReplaceAllString(r.TemplateString, "{{ \"{{ ."+VAR+".$1 }}\" }}") + templateStringReplaced := re.ReplaceAllString(r.templateString, "{{ \"{{ ."+VAR+".$1 }}\" }}") - tpl, err := r.tpl.Parse(templateString) + tpl, err := r.tpl.Parse(templateStringReplaced) if err != nil { return []byte{}, err } var buffer strings.Builder err = tpl.Execute(&buffer, map[string]interface{}{ - CONST: r.TemplateData.Constants}) + CONST: r.templateData.Constants}) if err != nil { return []byte{}, err } @@ -126,12 +119,12 @@ func (r *TemplateRenderer) ExecuteConstTemplate() ([]byte, error) { // used for compose operations func (r *TemplateRenderer) ExecuteNonSensitiveTemplate() ([]byte, error) { // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ "{{ var.KEY }}" }} - templateString := r.TemplateString + templateString := r.templateString re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) varMatches := re.FindAllStringSubmatch(templateString, -1) uniqueMatches := returnUniqueMatches(varMatches, 1) for k, matches := range uniqueMatches { - if _, ok := r.TemplateData.SensitiveVariables[matches[0]]; ok { + if _, ok := r.templateData.SensitiveVariables[matches[0]]; ok { templateString = strings.ReplaceAll(templateString, k, "{{ \""+k+"\" }}") } } @@ -143,8 +136,8 @@ func (r *TemplateRenderer) ExecuteNonSensitiveTemplate() ([]byte, error) { var buffer strings.Builder err = tpl.Execute(&buffer, map[string]interface{}{ - CONST: r.TemplateData.Constants, - VAR: r.TemplateData.Variables}) + CONST: r.templateData.Constants, + VAR: r.templateData.Variables}) if err != nil { return []byte{}, err } @@ -156,12 +149,12 @@ func (r *TemplateRenderer) ExecuteNonSensitiveTemplate() ([]byte, error) { // for display/printing only func (r *TemplateRenderer) ExecuteMaskedTemplate() ([]byte, error) { // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ var.KEY | mask }} - templateString := r.TemplateString + templateString := r.templateString re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) varMatches := re.FindAllStringSubmatch(templateString, -1) uniqueMatches := returnUniqueMatches(varMatches, 1) for k, matches := range uniqueMatches { - if _, ok := r.TemplateData.SensitiveVariables[matches[0]]; ok { + if _, ok := r.templateData.SensitiveVariables[matches[0]]; ok { templateString = strings.ReplaceAll(templateString, k, "********") } } @@ -173,8 +166,8 @@ func (r *TemplateRenderer) ExecuteMaskedTemplate() ([]byte, error) { var buffer strings.Builder err = tpl.Execute(&buffer, map[string]interface{}{ - CONST: r.TemplateData.Constants, - VAR: r.TemplateData.Variables}) + CONST: r.templateData.Constants, + VAR: r.templateData.Variables}) if err != nil { return []byte{}, err } diff --git a/src/internal/template/template_test.go b/src/internal/template/template_test.go index 4a917fbb..963dd543 100644 --- a/src/internal/template/template_test.go +++ b/src/internal/template/template_test.go @@ -10,10 +10,10 @@ import ( "github.com/defenseunicorns/lula/src/internal/template" ) -func testRender(t *testing.T, templateRenderer *template.TemplateRenderer, expected string) error { +func testRender(t *testing.T, templateRenderer *template.TemplateRenderer, renderType template.RenderType, expected string) error { t.Helper() - got, err := templateRenderer.Render() + got, err := templateRenderer.Render(renderType) if err != nil { return fmt.Errorf("error templating data: %v\n", err.Error()) } @@ -50,8 +50,8 @@ func TestExecuteFullTemplate(t *testing.T) { secret template: my-secret ` - tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) - err := testRender(t, tr, expected) + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.ALL, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -66,8 +66,8 @@ func TestExecuteFullTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) - err := testRender(t, tr, "") + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.ALL, "") if err == nil { t.Fatalf("Expected an error, but got nil") } @@ -81,8 +81,8 @@ func TestExecuteFullTemplate(t *testing.T) { constant template: {{ .constant.testVar }} ` - tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) - err := testRender(t, tr, "") + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.ALL, "") if err == nil { t.Fatal("expected error, got nil") } @@ -105,8 +105,8 @@ func TestExecuteFullTemplate(t *testing.T) { constant template: {{ .const.test-var }} ` - tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) - err := testRender(t, tr, "") + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.ALL, "") if err == nil { t.Fatal("expected error, got nil") } @@ -120,8 +120,8 @@ func TestExecuteFullTemplate(t *testing.T) { variable template: {{ .var.nokey.sub }} ` - tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) - err := testRender(t, tr, "") + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.ALL, "") if err == nil { t.Fatal("expected error, got nil") } @@ -133,8 +133,8 @@ func TestExecuteFullTemplate(t *testing.T) { templateString := ` constant template: {{ constant.testVar }} ` - tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) - err := testRender(t, tr, "") + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.ALL, "") if err == nil { t.Fatal("expected error, got nil") } @@ -153,8 +153,8 @@ func TestExecuteFullTemplate(t *testing.T) { expected := ` constant template: "one", "two", "three" ` - tr := template.NewTemplateRenderer(templateString, template.ALL, templateData) - err := testRender(t, tr, expected) + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.ALL, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -188,8 +188,8 @@ func TestExecuteConstTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, template.CONSTANTS, templateData) - err := testRender(t, tr, expected) + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.CONSTANTS, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -211,8 +211,8 @@ func TestExecuteConstTemplate(t *testing.T) { variable template: {{ .var.some_env_var }} secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, template.CONSTANTS, templateData) - err := testRender(t, tr, expected) + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.CONSTANTS, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -240,8 +240,8 @@ func TestExecuteConstTemplate(t *testing.T) { variable template: {{ .var.some_env_var }} secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, template.CONSTANTS, templateData) - err := testRender(t, tr, expected) + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.CONSTANTS, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -263,8 +263,8 @@ func TestExecuteConstTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, template.CONSTANTS, templateData) - err := testRender(t, tr, "") + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.CONSTANTS, "") if err == nil { t.Fatalf("Expected an error, but got nil") } @@ -302,8 +302,8 @@ func TestExecuteNonSensitiveTemplate(t *testing.T) { secret template2: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, template.NONSENSITIVE, templateData) - err := testRender(t, tr, expected) + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.NONSENSITIVE, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -332,8 +332,8 @@ func TestExecuteNonSensitiveTemplate(t *testing.T) { secret template: {{.var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, template.NONSENSITIVE, templateData) - err := testRender(t, tr, expected) + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.NONSENSITIVE, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -355,8 +355,8 @@ func TestExecuteNonSensitiveTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, template.NONSENSITIVE, templateData) - err := testRender(t, tr, "") + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.NONSENSITIVE, "") if err == nil { t.Fatalf("Expected an error, but got nil") } @@ -394,8 +394,8 @@ func TestExecuteMaskedTemplate(t *testing.T) { secret template2: ******** ` - tr := template.NewTemplateRenderer(templateString, template.MASKED, templateData) - err := testRender(t, tr, expected) + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.MASKED, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -424,8 +424,8 @@ func TestExecuteMaskedTemplate(t *testing.T) { secret template: ******** ` - tr := template.NewTemplateRenderer(templateString, template.MASKED, templateData) - err := testRender(t, tr, expected) + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.MASKED, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -447,8 +447,8 @@ func TestExecuteMaskedTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, template.MASKED, templateData) - err := testRender(t, tr, "") + tr := template.NewTemplateRenderer(templateString, templateData) + err := testRender(t, tr, template.MASKED, "") if err == nil { t.Fatalf("Expected an error, but got nil") } From 2a6fa1812d666e13e58b1a33889885fb86f0611c Mon Sep 17 00:00:00 2001 From: Megan Wolf Date: Fri, 20 Sep 2024 08:35:49 -0400 Subject: [PATCH 5/9] docs: updated config/template docs --- docs/getting-started/README.md | 6 +- docs/getting-started/configuration.md | 139 ++++++++++++++++++++++++++ docs/reference/configuration.md | 82 --------------- 3 files changed, 144 insertions(+), 83 deletions(-) create mode 100644 docs/getting-started/configuration.md delete mode 100644 docs/reference/configuration.md diff --git a/docs/getting-started/README.md b/docs/getting-started/README.md index 009bb49a..c4658cc5 100644 --- a/docs/getting-started/README.md +++ b/docs/getting-started/README.md @@ -63,4 +63,8 @@ See the following tutorials for some introductory lessons on how to use Lula. If Lula Validation manifests are the underlying mechanisms that dictates the evaluation of a system against a control as resulting in `satisfied` or `not satisfied`. A Lula Validation is linked to a control within a component definition via the OSCAL-specific property, [links](../oscal/oscal-validation-links.md). -Developing Lula Validations can sometimes be more art than science, but generally they should aim to be clear, concise, and robust to system changes. See our guide for [developing Lula Validations](./develop-a-validation.md) and the [references](../reference/README.md) for additional information. \ No newline at end of file +Developing Lula Validations can sometimes be more art than science, but generally they should aim to be clear, concise, and robust to system changes. See our guide for [developing Lula Validations](./develop-a-validation.md) and the [references](../reference/README.md) for additional information. + +### Configuration + +Lula supports the addition of a configuration file for specifying CLI flags and templating values. See our [configuration](./configuration.md) guide for more information. \ No newline at end of file diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md new file mode 100644 index 00000000..3df15a2d --- /dev/null +++ b/docs/getting-started/configuration.md @@ -0,0 +1,139 @@ +# Configuration + +Lula allows the use and specification of a config file in the following ways: +- Checking current working directory for a `lula-config.yaml` file +- Specification with environment variable `LULA_CONFIG=` + +Environment Variables can be used to specify configuration values through use of `LULA_` -> Example: `LULA_TARGET=il5` + +## Identification + +If identified, Lula will log which configuration file is used to stdout: +```bash +Using config file /home/dev/work/lula/lula-config.yaml +``` + +## Precedence + +The precedence for configuring settings, such as `target`, follows this hierarchy: + +### **Command Line Flag > Environment Variable > Configuration File** + +1. **Command Line Flag:** + When a setting like `target` is specified using a command line flag, this value takes the highest precedence, overriding any environment variable or configuration file settings. + +2. **Environment Variable:** + If the setting is not provided via a command line flag, an environment variable (e.g., `export LULA_TARGET=il5`) will take precedence over the configuration file. + +3. **Configuration File:** + In the absence of both a command line flag and environment variable, the value specified in the configuration file will be used. This will override system defaults. + +## Support + +Modification of command variables can be set in the configuration file: + +lula-config.yaml +```yaml +log_level: debug +target: il4 +summary: true +``` + +### Templating Configuration Fields + +Templating values are set in the configuration file via the use of `constants` and `variables` fields. + +#### Constants + +A sample `constants` section of a `lula-config.yaml` file is as follows: + +```yaml +constants: + type: software + title: lula + + resources: + name: test-pod-label + namespace: validation-test + imagelist: + - nginx + - nginx2 +``` + +Constants will respect the structure of a map[string]interface{} and can be referenced as follows: + +```yaml +# validaiton.yaml +metadata: + name: sample {{ .const.type }} validation for {{ .const.title }} +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: myPod + resource-rule: + name: {{ .const.resources.name }} + version: v1 + resource: pods + namespaces: [{{ .const.resources.namespace }}] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + validate if { + input.myPod.metadata.name == {{ .const.resources.name }} + input.myPod.containers[_].name in { {{ .const.resources.imagelist | concatToRegoList }} } + } +``` + +And will be rendered as: +```yaml +metadata: + name: sample software validation for lula +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: myPod + resource-rule: + name: myPod + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + validate if { + input.myPod.metadata.name == "myPod" + input.myPod.containers[_].image in { "nginx", "nginx2" } + } +``` + +The constant's keys should be in the format `.const.` and should not contain any '-' or '.' characters, as this will not respect the go text/template format. + +#### Variables + +A sample `variables` section of a `lula-config.yaml` file is as follows: + +```yaml +variables: + - key: some_lula_secret + sensitive: true + - key: some_env_var + default: this-should-be-overridden +``` + +The `variables` section is a list of `key`, `default`, and `sensitive` fields, where `sensitive` and `default` are optional. The `key` and `default` fields are strings, and the `sensitive` field is a boolean. + +A default value can be specified in the case where an environment variable may or may not be set, however an environment variable will always take precedence over a default value. + +The environment variable should follow the pattern of `LULA_VAR_` (not case sensitive), where `` is the key specified in the `variables` section. + +When using `sensitive` variables, the default behavior is to mask the value in the output of the template. \ No newline at end of file diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md deleted file mode 100644 index ea9fdbdf..00000000 --- a/docs/reference/configuration.md +++ /dev/null @@ -1,82 +0,0 @@ -# Configuration - -Lula allows the use and specification of a config file in the following ways: -- Checking current working directory for a `lula-config.yaml` file -- Specification with environment variable `LULA_CONFIG=` - -Environment Variables can be used to specify configuration values through use of `LULA_` -> Example: `LULA_TARGET=il5` - -## Identification - -If identified, Lula will log which configuration file is used to stdout: -```bash -Using config file /home/dev/work/lula/lula-config.yaml -``` - -## Precedence - -The precedence for configuring settings, such as `target`, follows this hierarchy: - -### **Command Line Flag > Environment Variable > Configuration File** - -1. **Command Line Flag:** - When a setting like `target` is specified using a command line flag, this value takes the highest precedence, overriding any environment variable or configuration file settings. - -2. **Environment Variable:** - If the setting is not provided via a command line flag, an environment variable (e.g., `export LULA_TARGET=il5`) will take precedence over the configuration file. - -3. **Configuration File:** - In the absence of both a command line flag and environment variable, the value specified in the configuration file will be used. This will override system defaults. - -## Support - -Modification of command variables can be set in the configuration file: - -lula-config.yaml -```yaml -log_level: debug -target: il4 -summary: true -``` - -### Templating Configuration Fields - -TODO - description of templating configuration fields - -```yaml -# constants = place to define non-changing values that can be of any type -# I think stuff here probably shouldn't be set by env vars - it's hard to be deterministic because of the character set differences, also type differences could lead to weird side effects -# Another note about this - we could probably easily pull in values of child components if this was referenced from a system-level - so this kind of behaves a bit like help values.yaml -constants: - # map[string]interface{} - elements referenced via template as {{ .const.key }} - type: software - title: lula - # Sample: Istio-specific values - istio: - namespace: istio-system # overriden by --set const.istio.namespace=my-istio-namespace - resources: - jsoncm: configmaps # (NOT) overriden by LULA_VAR_RESOURCES_JSONCM - # Problem with this is that json-cm and json_cm are different yaml keys, but would possibly reconcile to the same thing... so you're getting some side effects here that aren't great. - yamlcm: configmaps - secret: secrets - pod: pods - boolean: false # (NOT) overriden by LULA_VAR_RESOURCES_BOOLEAN - # ok how does this work when they're different types? an env var will always be a string... - exemptions: - - one - - two - - three - -# variables = place to define changing values of string type, and optionally sensitive values -# NOTE - if a variable is defined here, but does not have a default, you will need to make sure it's set either via --set or LULA_VAR_* for the template to execute without error (actually it doesn't error, just prints debug statements) -variables: - - key: some_lula_secret # set by LULA_VAR_SOME_LULA_SECRET / overriden by --set var.some_lula_secret=my-secret - default: blahblah # optional - sensitive: true # {{ var.some_lula_secret | mask }} - - key: some_env_var - default: this-should-be-overridden - -# Lula config values, still accessible via LULA_*, where * is the key -log_level: info -target: il5 -``` \ No newline at end of file From 999fcd8eda5df8b453f1834a67424f54e05f9137 Mon Sep 17 00:00:00 2001 From: Megan Wolf Date: Fri, 20 Sep 2024 08:53:05 -0400 Subject: [PATCH 6/9] fix: bump timeout in tui tests --- src/internal/tui/model_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/internal/tui/model_test.go b/src/internal/tui/model_test.go index 3c8ea47b..d4c0c106 100644 --- a/src/internal/tui/model_test.go +++ b/src/internal/tui/model_test.go @@ -15,6 +15,8 @@ import ( "github.com/muesli/termenv" ) +const timeout = time.Second * 10 + func init() { lipgloss.SetColorProfile(termenv.Ascii) tea.Sequence() @@ -49,7 +51,7 @@ func TestNewComponentDefinitionModel(t *testing.T) { t.Fatal("testModel is nil") } - fm := testModel.FinalModel(t, teatest.WithFinalTimeout(time.Second*5)) + fm := testModel.FinalModel(t, teatest.WithFinalTimeout(timeout)) teatest.RequireEqualOutput(t, []byte(fm.View())) } @@ -77,7 +79,7 @@ func TestMultiComponentDefinitionModel(t *testing.T) { t.Fatal("testModel is nil") } - fm := testModel.FinalModel(t, teatest.WithFinalTimeout(time.Second*5)) + fm := testModel.FinalModel(t, teatest.WithFinalTimeout(timeout)) teatest.RequireEqualOutput(t, []byte(fm.View())) } @@ -98,7 +100,7 @@ func TestNewAssessmentResultsModel(t *testing.T) { t.Fatal("testModel is nil") } - fm := testModel.FinalModel(t, teatest.WithFinalTimeout(time.Second*5)) + fm := testModel.FinalModel(t, teatest.WithFinalTimeout(timeout)) teatest.RequireEqualOutput(t, []byte(fm.View())) } @@ -124,7 +126,7 @@ func TestComponentControlSelect(t *testing.T) { t.Fatal("testModel is nil") } - fm := testModel.FinalModel(t, teatest.WithFinalTimeout(time.Second*5)) + fm := testModel.FinalModel(t, teatest.WithFinalTimeout(timeout)) teatest.RequireEqualOutput(t, []byte(fm.View())) } From d3e0c547f525483ac586bb1cf58fa2867ee4c02e Mon Sep 17 00:00:00 2001 From: Megan Wolf Date: Tue, 24 Sep 2024 06:36:39 -0400 Subject: [PATCH 7/9] fix: small refactor for extending tmpl --- src/cmd/common/common.go | 27 ++++++++++++++++++++ src/cmd/common/viper.go | 20 +++++++++++++++ src/cmd/tools/template.go | 41 +++++-------------------------- src/internal/template/template.go | 2 +- 4 files changed, 54 insertions(+), 36 deletions(-) create mode 100644 src/cmd/common/common.go diff --git a/src/cmd/common/common.go b/src/cmd/common/common.go new file mode 100644 index 00000000..8e753859 --- /dev/null +++ b/src/cmd/common/common.go @@ -0,0 +1,27 @@ +package common + +import ( + "fmt" + "strings" + + "github.com/defenseunicorns/lula/src/internal/template" + "github.com/defenseunicorns/lula/src/pkg/message" +) + +func ParseTemplateOverrides(setFlags []string) map[string]string { + overrides := make(map[string]string) + for _, flag := range setFlags { + parts := strings.SplitN(flag, "=", 2) + if len(parts) != 2 { + message.Fatalf(fmt.Errorf("invalid --set flag format, should be .root.key=value"), "invalid --set flag format, should be .root.key=value") + } + + if !strings.HasPrefix(parts[0], "."+template.CONST+".") && !strings.HasPrefix(parts[0], "."+template.VAR+".") { + message.Fatalf(fmt.Errorf("invalid --set flag format, path should start with .const or .var"), "invalid --set flag format, path should start with .const or .var") + } + + path, value := parts[0], parts[1] + overrides[path] = value + } + return overrides +} diff --git a/src/cmd/common/viper.go b/src/cmd/common/viper.go index 7d3ea027..8763620b 100644 --- a/src/cmd/common/viper.go +++ b/src/cmd/common/viper.go @@ -2,9 +2,11 @@ package common import ( "errors" + "fmt" "os" "strings" + "github.com/defenseunicorns/lula/src/internal/template" "github.com/defenseunicorns/lula/src/pkg/message" "github.com/spf13/viper" ) @@ -72,6 +74,24 @@ func GetViper() *viper.Viper { return v } +// GetTemplateConfig loads the constants and variables from the viper config +func GetTemplateConfig() (map[string]interface{}, []template.VariableConfig, error) { + var constants map[string]interface{} + var variables []template.VariableConfig + + err := v.UnmarshalKey(VConstants, &constants) + if err != nil { + return nil, nil, fmt.Errorf("unable to unmarshal constants into map: %v", err) + } + + err = v.UnmarshalKey(VVariables, &variables) + if err != nil { + return nil, nil, fmt.Errorf("unable to unmarshal variables into slice: %v", err) + } + + return constants, variables, nil +} + func isVersionCmd() bool { args := os.Args return len(args) > 1 && (args[1] == "version" || args[1] == "v") diff --git a/src/cmd/tools/template.go b/src/cmd/tools/template.go index 55c8b512..6ae1b1cf 100644 --- a/src/cmd/tools/template.go +++ b/src/cmd/tools/template.go @@ -45,9 +45,6 @@ func TemplateCommand() *cobra.Command { Args: cobra.NoArgs, Example: templateHelp, RunE: func(cmd *cobra.Command, args []string) error { - // Get current viper pointer - v := common.GetViper() - // Read file data, err := pkgCommon.ReadFileToBytes(inputFile) if err != nil { @@ -55,27 +52,19 @@ func TemplateCommand() *cobra.Command { } // Validate render type - renderType, err := getRenderType(renderTypeString) + renderType, err := parseRenderType(renderTypeString) if err != nil { - message.Warn("invalid render type, defaulting to masked") + message.Warnf("invalid render type, defaulting to masked: %v", err) } // Get constants and variables for templating from viper config - var constants map[string]interface{} - var variables []template.VariableConfig - - err = v.UnmarshalKey(common.VConstants, &constants) - if err != nil { - return fmt.Errorf("unable to unmarshal constants into map: %v", err) - } - - err = v.UnmarshalKey(common.VVariables, &variables) + constants, variables, err := common.GetTemplateConfig() if err != nil { - return fmt.Errorf("unable to unmarshal variables into slice: %v", err) + return fmt.Errorf("error getting template config: %v", err) } // Get overrides from --set flag - overrides := getOverrides(setOpts) + overrides := common.ParseTemplateOverrides(setOpts) // Handles merging viper config file data + environment variables // Throws an error if config keys are invalid for templating @@ -123,7 +112,7 @@ func init() { toolsCmd.AddCommand(TemplateCommand()) } -func getRenderType(item string) (template.RenderType, error) { +func parseRenderType(item string) (template.RenderType, error) { switch strings.ToLower(item) { case "masked": return template.MASKED, nil @@ -136,21 +125,3 @@ func getRenderType(item string) (template.RenderType, error) { } return template.MASKED, fmt.Errorf("invalid render type: %s", item) } - -func getOverrides(setFlags []string) map[string]string { - overrides := make(map[string]string) - for _, flag := range setFlags { - parts := strings.SplitN(flag, "=", 2) - if len(parts) != 2 { - message.Fatalf(fmt.Errorf("invalid --set flag format, should be .root.key=value"), "invalid --set flag format, should be .root.key=value") - } - - if !strings.HasPrefix(parts[0], "."+template.CONST+".") && !strings.HasPrefix(parts[0], "."+template.VAR+".") { - message.Fatalf(fmt.Errorf("invalid --set flag format, path should start with .const or .var"), "invalid --set flag format, path should start with .const or .var") - } - - path, value := parts[0], parts[1] - overrides[path] = value - } - return overrides -} diff --git a/src/internal/template/template.go b/src/internal/template/template.go index 5bae9f57..6bd34d76 100644 --- a/src/internal/template/template.go +++ b/src/internal/template/template.go @@ -50,7 +50,7 @@ func (r *TemplateRenderer) Render(t RenderType) ([]byte, error) { case ALL: return r.ExecuteFullTemplate() default: - return r.ExecuteMaskedTemplate() + return []byte{}, fmt.Errorf("invalid render type: %s", t) } } From 202849a399c09c2ba249e9ee977a91055885f0f7 Mon Sep 17 00:00:00 2001 From: Megan Wolf Date: Tue, 24 Sep 2024 08:58:33 -0400 Subject: [PATCH 8/9] fix: bumped timeout --- src/internal/tui/model_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/tui/model_test.go b/src/internal/tui/model_test.go index eb2d72e2..89fa7e77 100644 --- a/src/internal/tui/model_test.go +++ b/src/internal/tui/model_test.go @@ -14,7 +14,7 @@ import ( "github.com/muesli/termenv" ) -const timeout = time.Second * 10 +const timeout = time.Second * 20 func init() { lipgloss.SetColorProfile(termenv.Ascii) From 579404d574ccdbc6ade4fe7d024c2c1cc0bb924f Mon Sep 17 00:00:00 2001 From: Megan Wolf Date: Wed, 25 Sep 2024 13:31:25 -0400 Subject: [PATCH 9/9] fix: updated docs, slightly modified templaterenderer struct --- docs/cli-commands/lula_tools_template.md | 13 +++- src/cmd/common/common.go | 9 ++- src/cmd/tools/template.go | 11 ++-- src/internal/template/template.go | 38 +++++------- src/internal/template/template_test.go | 72 +++++++++++----------- src/test/e2e/standard/testdata/help.golden | 2 +- 6 files changed, 76 insertions(+), 69 deletions(-) diff --git a/docs/cli-commands/lula_tools_template.md b/docs/cli-commands/lula_tools_template.md index d7a68747..70e43ec4 100644 --- a/docs/cli-commands/lula_tools_template.md +++ b/docs/cli-commands/lula_tools_template.md @@ -19,13 +19,20 @@ lula tools template [flags] ``` -To template an OSCAL Model: +To template an OSCAL Model, defaults to masking sensitive variables: lula tools template -f ./oscal-component.yaml To indicate a specific output file: lula tools template -f ./oscal-component.yaml -o templated-oscal-component.yaml -Data for the templating should be stored under the 'variables' configuration item in a lula-config.yaml file +To perform overrides on the template data: + lula tools template -f ./oscal-component.yaml --set .var.key1=value1 --set .const.key2=value2 + +To perform the full template operation, including sensitive data: + lula tools template -f ./oscal-component.yaml --render all + +Data for templating should be stored under 'constants' or 'variables' configuration items in a lula-config.yaml file +See documentation for more detail on configuration schema ``` @@ -35,6 +42,8 @@ Data for the templating should be stored under the 'variables' configuration ite -h, --help help for template -f, --input-file string the path to the target artifact -o, --output-file string the path to the output file. If not specified, the output file will be directed to stdout + -r, --render string values to render the template with, options are: masked, constants, non-sensitive, all (default "masked") + -s, --set strings set a value in the template data ``` ### Options inherited from parent commands diff --git a/src/cmd/common/common.go b/src/cmd/common/common.go index 8e753859..3d66ceca 100644 --- a/src/cmd/common/common.go +++ b/src/cmd/common/common.go @@ -5,23 +5,22 @@ import ( "strings" "github.com/defenseunicorns/lula/src/internal/template" - "github.com/defenseunicorns/lula/src/pkg/message" ) -func ParseTemplateOverrides(setFlags []string) map[string]string { +func ParseTemplateOverrides(setFlags []string) (map[string]string, error) { overrides := make(map[string]string) for _, flag := range setFlags { parts := strings.SplitN(flag, "=", 2) if len(parts) != 2 { - message.Fatalf(fmt.Errorf("invalid --set flag format, should be .root.key=value"), "invalid --set flag format, should be .root.key=value") + return overrides, fmt.Errorf("invalid --set flag format, should be .root.key=value") } if !strings.HasPrefix(parts[0], "."+template.CONST+".") && !strings.HasPrefix(parts[0], "."+template.VAR+".") { - message.Fatalf(fmt.Errorf("invalid --set flag format, path should start with .const or .var"), "invalid --set flag format, path should start with .const or .var") + return overrides, fmt.Errorf("invalid --set flag format, path should start with .const or .var") } path, value := parts[0], parts[1] overrides[path] = value } - return overrides + return overrides, nil } diff --git a/src/cmd/tools/template.go b/src/cmd/tools/template.go index 6ae1b1cf..4b1520f1 100644 --- a/src/cmd/tools/template.go +++ b/src/cmd/tools/template.go @@ -21,7 +21,7 @@ To indicate a specific output file: lula tools template -f ./oscal-component.yaml -o templated-oscal-component.yaml To perform overrides on the template data: - lula tools template -f ./oscal-component.yaml --set var.key1=value1 --set const.key2=value2 + lula tools template -f ./oscal-component.yaml --set .var.key1=value1 --set .const.key2=value2 To perform the full template operation, including sensitive data: lula tools template -f ./oscal-component.yaml --render all @@ -64,7 +64,10 @@ func TemplateCommand() *cobra.Command { } // Get overrides from --set flag - overrides := common.ParseTemplateOverrides(setOpts) + overrides, err := common.ParseTemplateOverrides(setOpts) + if err != nil { + return fmt.Errorf("error parsing template overrides: %v", err) + } // Handles merging viper config file data + environment variables // Throws an error if config keys are invalid for templating @@ -73,8 +76,8 @@ func TemplateCommand() *cobra.Command { return fmt.Errorf("error collecting templating data: %v", err) } - templateRenderer := template.NewTemplateRenderer(string(data), templateData) - output, err := templateRenderer.Render(renderType) + templateRenderer := template.NewTemplateRenderer(templateData) + output, err := templateRenderer.Render(string(data), renderType) if err != nil { return fmt.Errorf("error rendering template: %v", err) } diff --git a/src/internal/template/template.go b/src/internal/template/template.go index 6bd34d76..865c4e46 100644 --- a/src/internal/template/template.go +++ b/src/internal/template/template.go @@ -26,29 +26,27 @@ const ( ) type TemplateRenderer struct { - tpl *template.Template - templateString string - templateData *TemplateData + tpl *template.Template + templateData *TemplateData } -func NewTemplateRenderer(templateString string, templateData *TemplateData) *TemplateRenderer { +func NewTemplateRenderer(templateData *TemplateData) *TemplateRenderer { return &TemplateRenderer{ - tpl: createTemplate(), - templateString: templateString, - templateData: templateData, + tpl: createTemplate(), + templateData: templateData, } } -func (r *TemplateRenderer) Render(t RenderType) ([]byte, error) { +func (r *TemplateRenderer) Render(templateString string, t RenderType) ([]byte, error) { switch t { case MASKED: - return r.ExecuteMaskedTemplate() + return r.ExecuteMaskedTemplate(templateString) case CONSTANTS: - return r.ExecuteConstTemplate() + return r.ExecuteConstTemplate(templateString) case NONSENSITIVE: - return r.ExecuteNonSensitiveTemplate() + return r.ExecuteNonSensitiveTemplate(templateString) case ALL: - return r.ExecuteFullTemplate() + return r.ExecuteFullTemplate(templateString) default: return []byte{}, fmt.Errorf("invalid render type: %s", t) } @@ -75,8 +73,8 @@ type VariableConfig struct { } // ExecuteFullTemplate templates everything -func (r *TemplateRenderer) ExecuteFullTemplate() ([]byte, error) { - tpl, err := r.tpl.Parse(r.templateString) +func (r *TemplateRenderer) ExecuteFullTemplate(templateString string) ([]byte, error) { + tpl, err := r.tpl.Parse(templateString) if err != nil { return []byte{}, err } @@ -95,12 +93,12 @@ func (r *TemplateRenderer) ExecuteFullTemplate() ([]byte, error) { // ExecuteConstTemplate templates only constants // this templates only values in the constants map -func (r *TemplateRenderer) ExecuteConstTemplate() ([]byte, error) { +func (r *TemplateRenderer) ExecuteConstTemplate(templateString string) ([]byte, error) { // Find anything {{ var.KEY }} and replace with {{ "{{ var.KEY }}" }} re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) - templateStringReplaced := re.ReplaceAllString(r.templateString, "{{ \"{{ ."+VAR+".$1 }}\" }}") + templateString = re.ReplaceAllString(templateString, "{{ \"{{ ."+VAR+".$1 }}\" }}") - tpl, err := r.tpl.Parse(templateStringReplaced) + tpl, err := r.tpl.Parse(templateString) if err != nil { return []byte{}, err } @@ -117,9 +115,8 @@ func (r *TemplateRenderer) ExecuteConstTemplate() ([]byte, error) { // ExecuteNonSensitiveTemplate templates only constants and non-sensitive variables // used for compose operations -func (r *TemplateRenderer) ExecuteNonSensitiveTemplate() ([]byte, error) { +func (r *TemplateRenderer) ExecuteNonSensitiveTemplate(templateString string) ([]byte, error) { // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ "{{ var.KEY }}" }} - templateString := r.templateString re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) varMatches := re.FindAllStringSubmatch(templateString, -1) uniqueMatches := returnUniqueMatches(varMatches, 1) @@ -147,9 +144,8 @@ func (r *TemplateRenderer) ExecuteNonSensitiveTemplate() ([]byte, error) { // ExecuteMaskedTemplate templates all values, but masks the sensitive ones // for display/printing only -func (r *TemplateRenderer) ExecuteMaskedTemplate() ([]byte, error) { +func (r *TemplateRenderer) ExecuteMaskedTemplate(templateString string) ([]byte, error) { // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ var.KEY | mask }} - templateString := r.templateString re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) varMatches := re.FindAllStringSubmatch(templateString, -1) uniqueMatches := returnUniqueMatches(varMatches, 1) diff --git a/src/internal/template/template_test.go b/src/internal/template/template_test.go index 963dd543..63fcc2a7 100644 --- a/src/internal/template/template_test.go +++ b/src/internal/template/template_test.go @@ -10,10 +10,10 @@ import ( "github.com/defenseunicorns/lula/src/internal/template" ) -func testRender(t *testing.T, templateRenderer *template.TemplateRenderer, renderType template.RenderType, expected string) error { +func testRender(t *testing.T, templateRenderer *template.TemplateRenderer, templateString string, renderType template.RenderType, expected string) error { t.Helper() - got, err := templateRenderer.Render(renderType) + got, err := templateRenderer.Render(templateString, renderType) if err != nil { return fmt.Errorf("error templating data: %v\n", err.Error()) } @@ -50,8 +50,8 @@ func TestExecuteFullTemplate(t *testing.T) { secret template: my-secret ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.ALL, expected) + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -66,8 +66,8 @@ func TestExecuteFullTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.ALL, "") + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") if err == nil { t.Fatalf("Expected an error, but got nil") } @@ -81,8 +81,8 @@ func TestExecuteFullTemplate(t *testing.T) { constant template: {{ .constant.testVar }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.ALL, "") + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") if err == nil { t.Fatal("expected error, got nil") } @@ -105,8 +105,8 @@ func TestExecuteFullTemplate(t *testing.T) { constant template: {{ .const.test-var }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.ALL, "") + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") if err == nil { t.Fatal("expected error, got nil") } @@ -120,8 +120,8 @@ func TestExecuteFullTemplate(t *testing.T) { variable template: {{ .var.nokey.sub }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.ALL, "") + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") if err == nil { t.Fatal("expected error, got nil") } @@ -133,8 +133,8 @@ func TestExecuteFullTemplate(t *testing.T) { templateString := ` constant template: {{ constant.testVar }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.ALL, "") + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") if err == nil { t.Fatal("expected error, got nil") } @@ -153,8 +153,8 @@ func TestExecuteFullTemplate(t *testing.T) { expected := ` constant template: "one", "two", "three" ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.ALL, expected) + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -188,8 +188,8 @@ func TestExecuteConstTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.CONSTANTS, expected) + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.CONSTANTS, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -211,8 +211,8 @@ func TestExecuteConstTemplate(t *testing.T) { variable template: {{ .var.some_env_var }} secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.CONSTANTS, expected) + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.CONSTANTS, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -240,8 +240,8 @@ func TestExecuteConstTemplate(t *testing.T) { variable template: {{ .var.some_env_var }} secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.CONSTANTS, expected) + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.CONSTANTS, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -263,8 +263,8 @@ func TestExecuteConstTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.CONSTANTS, "") + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.CONSTANTS, "") if err == nil { t.Fatalf("Expected an error, but got nil") } @@ -302,8 +302,8 @@ func TestExecuteNonSensitiveTemplate(t *testing.T) { secret template2: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.NONSENSITIVE, expected) + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.NONSENSITIVE, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -332,8 +332,8 @@ func TestExecuteNonSensitiveTemplate(t *testing.T) { secret template: {{.var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.NONSENSITIVE, expected) + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.NONSENSITIVE, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -355,8 +355,8 @@ func TestExecuteNonSensitiveTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.NONSENSITIVE, "") + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.NONSENSITIVE, "") if err == nil { t.Fatalf("Expected an error, but got nil") } @@ -394,8 +394,8 @@ func TestExecuteMaskedTemplate(t *testing.T) { secret template2: ******** ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.MASKED, expected) + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.MASKED, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -424,8 +424,8 @@ func TestExecuteMaskedTemplate(t *testing.T) { secret template: ******** ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.MASKED, expected) + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.MASKED, expected) if err != nil { t.Fatalf("Expected no error, but got %v", err) } @@ -447,8 +447,8 @@ func TestExecuteMaskedTemplate(t *testing.T) { secret template: {{ .var.some_lula_secret }} ` - tr := template.NewTemplateRenderer(templateString, templateData) - err := testRender(t, tr, template.MASKED, "") + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.MASKED, "") if err == nil { t.Fatalf("Expected an error, but got nil") } diff --git a/src/test/e2e/standard/testdata/help.golden b/src/test/e2e/standard/testdata/help.golden index 535327d8..4b858bed 100644 --- a/src/test/e2e/standard/testdata/help.golden +++ b/src/test/e2e/standard/testdata/help.golden @@ -12,7 +12,7 @@ To indicate a specific output file: lula tools template -f ./oscal-component.yaml -o templated-oscal-component.yaml To perform overrides on the template data: - lula tools template -f ./oscal-component.yaml --set var.key1=value1 --set const.key2=value2 + lula tools template -f ./oscal-component.yaml --set .var.key1=value1 --set .const.key2=value2 To perform the full template operation, including sensitive data: lula tools template -f ./oscal-component.yaml --render all