Initial start of big refactor to increase testing

.changes/unreleased/Refactor-20230831-194854.yaml

@@ -0,0 +1,3 @@
+kind: Refactor
+body: Refactor the entire codebase to be easier to test and update
+time: 2023-08-31T19:48:54.653125-05:00

.gitignore

@@ -1,3 +1,4 @@
 /kubectl-opslevel
 src/dist
 .idea
+.DS_Store

src/cmd/collect.go

@@ -6,19 +6,13 @@ import (
 	"time"
 
 	"github.com/opslevel/kubectl-opslevel/common"
-	"github.com/opslevel/kubectl-opslevel/config"
-	"github.com/opslevel/kubectl-opslevel/jq"
 	"github.com/opslevel/kubectl-opslevel/k8sutils"
+	"github.com/opslevel/kubectl-opslevel/pkg/config"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
-var (
-	collectResyncInterval int
-	collectBatchSize      int
-)
-
 var collectCmd = &cobra.Command{
 	Use:   "collect",
 	Short: "Acts as a kubernetes controller to collect resources for submission to OpsLevel as custom event check payloads",
@@ -27,20 +21,15 @@ var collectCmd = &cobra.Command{
 }
 
 func init() {
-	rootCmd.AddCommand(collectCmd)
+	runCmd.AddCommand(collectCmd)
 
 	collectCmd.Flags().StringP("integration-url", "i", "", "OpsLevel integration url (OPSLEVEL_INTEGRATION_URL)")
-	collectCmd.Flags().IntVar(&collectResyncInterval, "resync", 24, "The amount (in hours) before a full resync of the kubernetes cluster happens with OpsLevel. [default: 24]")
-	collectCmd.Flags().IntVar(&collectBatchSize, "batch", 500, "The max amount of k8s resources to batch process. Helps to speedup initial startup. [default: 500]")
 
 	viper.BindEnv("integration-url", "OPSLEVEL_INTEGRATION_URL")
 }
 
 func runCollect(cmd *cobra.Command, args []string) {
-	config, configErr := config.New()
-	cobra.CheckErr(configErr)
-
-	jq.ValidateInstalled()
+	config := getCfgFile()
 
 	integrationUrl := viper.GetString("integration-url")
 	if len(integrationUrl) <= 0 {
@@ -49,7 +38,7 @@ func runCollect(cmd *cobra.Command, args []string) {
 
 	k8sClient := k8sutils.CreateKubernetesClient()
 
-	resync := time.Hour * time.Duration(reconcileResyncInterval)
+	resync := time.Hour * time.Duration(resyncInterval)
 	collectQueue := make(chan string, 1)
 
 	for i, importConfig := range config.Service.Collect {
@@ -64,7 +53,7 @@ func runCollect(cmd *cobra.Command, args []string) {
 			continue
 		}
 		callback := createCollectHandler(fmt.Sprintf("service.import[%d]", i), importConfig, collectQueue)
-		controller := k8sutils.NewController(*gvr, resync, reconcileBatchSize)
+		controller := k8sutils.NewController(*gvr, resync, batchSize)
 		controller.OnAdd = callback
 		controller.OnUpdate = callback
 		go controller.Start(1)

src/cmd/config.go

@@ -1,99 +1,17 @@
 package cmd
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 
 	yaml "gopkg.in/yaml.v3"
 
 	"github.com/alecthomas/jsonschema"
-	"github.com/opslevel/kubectl-opslevel/config"
+	"github.com/opslevel/kubectl-opslevel/pkg/config"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
-// Make sure we only use spaces inside of these samples
-var configSimple = []byte(`#Simple Opslevel CLI Config
-version: "1.2.0"
-service:
-  import:
-    - selector: # This limits what data we look at in Kubernetes
-        apiVersion: apps/v1 # only supports resources found in 'kubectl api-resources --verbs="get,list"'
-        kind: Deployment
-        excludes: # filters out resources if any expression returns truthy
-          - .metadata.namespace == "kube-system"
-          - .metadata.annotations."opslevel.com/ignore"
-      opslevel: # This is how you map your kubernetes data to opslevel service
-        name: .metadata.name
-        owner: .metadata.namespace
-        aliases: # This are how we identify the services again during reconciliation - please make sure they are very unique
-          - '"k8s:\(.metadata.name)-\(.metadata.namespace)"'
-        tags:
-          assign: # tag with the same key name but with a different value will be updated on the service
-            - '{"imported": "kubectl-opslevel"}'
-            - .metadata.labels
-          create: # tag with the same key name but with a different value with be added to the service
-            - '{"environment": .spec.template.metadata.labels.environment}'
-  collect:
-    - selector: # This limits what data we look at in Kubernetes
-        apiVersion: apps/v1 # only supports resources found in 'kubectl api-resources --verbs="get,list"'
-        kind: Deployment
-        excludes: # filters out resources if any expression returns truthy
-          - .metadata.namespace == "kube-system"
-          - .metadata.annotations."opslevel.com/ignore"
-`)
-
-var configSample = []byte(`#Sample Opslevel CLI Config
-version: "1.2.0"
-service:
-  import:
-    - selector: # This limits what data we look at in Kubernetes
-        apiVersion: apps/v1 # only supports resources found in 'kubectl api-resources --verbs="get,list"'
-        kind: Deployment
-        excludes: # filters out resources if any expression returns truthy
-          - .metadata.namespace == "kube-system"
-          - .metadata.annotations."opslevel.com/ignore"
-      opslevel: # This is how you map your kubernetes data to opslevel service
-        name: .metadata.name
-        description: .metadata.annotations."opslevel.com/description"
-        owner: .metadata.annotations."opslevel.com/owner"
-        lifecycle: .metadata.annotations."opslevel.com/lifecycle"
-        tier: .metadata.annotations."opslevel.com/tier"
-        product: .metadata.annotations."opslevel.com/product"
-        language: .metadata.annotations."opslevel.com/language"
-        framework: .metadata.annotations."opslevel.com/framework"
-        aliases: # This are how we identify the services again during reconciliation - please make sure they are very unique
-          - '"k8s:\(.metadata.name)-\(.metadata.namespace)"'
-        tags:
-          assign: # tag with the same key name but with a different value will be updated on the service
-            - '{"imported": "kubectl-opslevel"}'
-            # find annoations with format: opslevel.com/tags.<key name>: <value>
-            - '.metadata.annotations | to_entries |  map(select(.key | startswith("opslevel.com/tags"))) | map({(.key | split(".")[2]): .value})'
-            - .metadata.labels
-          create: # tag with the same key name but with a different value with be added to the service
-            - '{"environment": .spec.template.metadata.labels.environment}'
-        tools:
-          - '{"category": "other", "environment": "production", "displayName": "my-cool-tool", "url": .metadata.annotations."example.com/my-cool-tool"} | if .url then . else empty end'
-          # find annotations with format: opslevel.com/tools.<category>.<displayname>: <url> 
-          - '.metadata.annotations | to_entries |  map(select(.key | startswith("opslevel.com/tools"))) | map({"category": .key | split(".")[2], "displayName": .key | split(".")[3], "url": .value})'
-          # OR find annotations with format: opslevel.com/tools.<category>.<environment>.<displayname>: <url> 
-          # - '.metadata.annotations | to_entries |  map(select(.key | startswith("opslevel.com/tools"))) | map({"category": .key | split(".")[2], "environment": .key | split(".")[3], "displayName": .key | split(".")[4], "url": .value})'
-        repositories: # attach repositories to the service using the opslevel repo alias - IE github.com:hashicorp/vault
-          - '{"name": "My Cool Repo", "directory": "/", "repo": .metadata.annotations.repo} | if .repo then . else empty end'
-          # if just the alias is returned as a single string we'll build the name for you and set the directory to "/"
-          - .metadata.annotations.repo
-          # find annotations with format: opslevel.com/repo.<displayname>.<repo.subpath.dots.turned.to.forwardslash>: <opslevel repo alias> 
-          - '.metadata.annotations | to_entries |  map(select(.key | startswith("opslevel.com/repos"))) | map({"name": .key | split(".")[2], "directory": .key | split(".")[3:] | join("/"), "repo": .value})'
-  collect:
-    - selector: # This limits what data we look at in Kubernetes
-        apiVersion: apps/v1 # only supports resources found in 'kubectl api-resources --verbs="get,list"'
-        kind: Deployment
-        excludes: # filters out resources if any expression returns truthy
-          - .metadata.namespace == "kube-system"
-          - .metadata.annotations."opslevel.com/ignore"
-`)
-
 var configCmd = &cobra.Command{
 	Use:   "config",
 	Short: "Commands for working with the opslevel configuration",
@@ -117,8 +35,7 @@ var configViewCmd = &cobra.Command{
 	Short: "Print the final configuration result",
 	Long:  "Print the final configuration after loading all the overrides and defaults",
 	Run: func(cmd *cobra.Command, args []string) {
-		conf, err := config.New()
-		cobra.CheckErr(err)
+		conf := getCfgFile()
 		output, err2 := yaml.Marshal(conf)
 		cobra.CheckErr(err2)
 		fmt.Println(string(output))
@@ -130,7 +47,7 @@ var configSampleCmd = &cobra.Command{
 	Short: "Print a sample config file",
 	Long:  "Print a sample config file which could be used",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println(getSample(viper.GetBool("simple")))
+		fmt.Println(config.GetSample(viper.GetBool("simple")))
 	},
 }
 
@@ -145,26 +62,3 @@ func init() {
 	configSampleCmd.Flags().Bool("simple", false, "Adjust the sample config to a bit simpler")
 	viper.BindPFlags(configSampleCmd.Flags())
 }
-
-func getSample(simple bool) string {
-	var sample []byte
-	if simple == true {
-		sample = configSimple
-	} else {
-		sample = configSample
-	}
-	// we use yaml unmarshal to prove that the samples are valid yaml
-	var nodes yaml.Node
-	unmarshalErr := yaml.Unmarshal(sample, &nodes)
-	if unmarshalErr != nil {
-		return unmarshalErr.Error()
-	}
-	var b bytes.Buffer
-	yamlEncoder := yaml.NewEncoder(&b)
-	yamlEncoder.SetIndent(2)
-	encodeErr := yamlEncoder.Encode(&nodes)
-	if encodeErr != nil {
-		return encodeErr.Error()
-	}
-	return string(b.Bytes())
-}

src/cmd/import.go

@@ -4,8 +4,6 @@ import (
 	"sync"
 
 	"github.com/opslevel/kubectl-opslevel/common"
-	"github.com/opslevel/kubectl-opslevel/config"
-	"github.com/opslevel/kubectl-opslevel/jq"
 	"github.com/opslevel/opslevel-go/v2023"
 
 	"github.com/rs/zerolog/log"
@@ -24,10 +22,7 @@ func init() {
 }
 
 func runImport(cmd *cobra.Command, args []string) {
-	config, configErr := config.New()
-	cobra.CheckErr(configErr)
-
-	jq.ValidateInstalled()
+	config := getCfgFile()
 
 	services, servicesErr := common.GetAllServices(config)
 	cobra.CheckErr(servicesErr)

src/cmd/preview.go

@@ -9,9 +9,6 @@ import (
 	"time"
 
 	"github.com/opslevel/kubectl-opslevel/common"
-	"github.com/opslevel/kubectl-opslevel/config"
-	"github.com/opslevel/kubectl-opslevel/jq"
-
 	_ "github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 )
@@ -39,10 +36,7 @@ func runPreview(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	config, err := config.New()
-	cobra.CheckErr(err)
-
-	jq.ValidateInstalled()
+	config := getCfgFile()
 
 	services, err2 := common.GetAllServices(config)
 	cobra.CheckErr(err2)

src/cmd/reconcile.go

@@ -6,19 +6,13 @@ import (
 	"time"
 
 	"github.com/opslevel/kubectl-opslevel/common"
-	"github.com/opslevel/kubectl-opslevel/config"
-	"github.com/opslevel/kubectl-opslevel/jq"
 	"github.com/opslevel/kubectl-opslevel/k8sutils"
+	"github.com/opslevel/kubectl-opslevel/pkg/config"
 	"github.com/opslevel/opslevel-go/v2023"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 )
 
-var (
-	reconcileResyncInterval int
-	reconcileBatchSize      int
-)
-
 var reconcileCmd = &cobra.Command{
 	Use:   "reconcile",
 	Short: "Run in the foreground as a kubernetes controller to reconcile data with service entries in OpsLevel",
@@ -27,17 +21,11 @@ var reconcileCmd = &cobra.Command{
 }
 
 func init() {
-	rootCmd.AddCommand(reconcileCmd)
-
-	reconcileCmd.Flags().IntVar(&reconcileResyncInterval, "resync", 24, "The amount (in hours) before a full resync of the kubernetes cluster happens with OpsLevel. [default: 24]")
-	reconcileCmd.Flags().IntVar(&reconcileBatchSize, "batch", 500, "The max amount of k8s resources to batch process with jq. Helps to speedup initial startup. [default: 500]")
+	runCmd.AddCommand(reconcileCmd)
 }
 
 func runReconcile(cmd *cobra.Command, args []string) {
-	config, configErr := config.New()
-	cobra.CheckErr(configErr)
-
-	jq.ValidateInstalled()
+	config := getCfgFile()
 
 	k8sClient := k8sutils.CreateKubernetesClient()
 	olClient := createOpslevelClient()
@@ -46,7 +34,7 @@ func runReconcile(cmd *cobra.Command, args []string) {
 	opslevel.Cache.CacheLifecycles(olClient)
 	opslevel.Cache.CacheTeams(olClient)
 
-	resync := time.Hour * time.Duration(reconcileResyncInterval)
+	resync := time.Hour * time.Duration(resyncInterval)
 	reconcileQueue := make(chan common.ServiceRegistration, 1)
 
 	for i, importConfig := range config.Service.Import {
@@ -61,7 +49,7 @@ func runReconcile(cmd *cobra.Command, args []string) {
 			continue
 		}
 		callback := createHandler(fmt.Sprintf("service.import[%d]", i), importConfig, reconcileQueue)
-		controller := k8sutils.NewController(*gvr, resync, reconcileBatchSize)
+		controller := k8sutils.NewController(*gvr, resync, batchSize)
 		controller.OnAdd = callback
 		controller.OnUpdate = callback
 		go controller.Start(1)

src/cmd/root.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/go-resty/resty/v2"
+	"github.com/opslevel/kubectl-opslevel/pkg/config"
 	"os"
 	"runtime"
 	"strings"
@@ -70,23 +71,6 @@ func initConfig() {
 }
 
 func readConfig() {
-	if cfgFile != "" {
-		if cfgFile == "." {
-			viper.SetConfigType("yaml")
-			viper.ReadConfig(os.Stdin)
-			return
-		} else {
-			viper.SetConfigFile(cfgFile)
-		}
-	} else {
-		home, err := os.UserHomeDir()
-		cobra.CheckErr(err)
-
-		viper.SetConfigName("opslevel")
-		viper.SetConfigType("yaml")
-		viper.AddConfigPath(".")
-		viper.AddConfigPath(home)
-	}
 	viper.SetEnvPrefix("OPSLEVEL")
 	viper.AutomaticEnv()
 	viper.ReadInConfig()
@@ -179,3 +163,11 @@ func createOpslevelClient() *opslevel.Client {
 func createRestClient() *resty.Client {
 	return opslevel.NewRestClient(opslevel.SetURL(viper.GetString("api-url")))
 }
+
+func getCfgFile() *config.Config {
+	data, err := os.ReadFile(cfgFile)
+	cobra.CheckErr(err)
+	output, err := config.NewConfig(data)
+	cobra.CheckErr(err)
+	return output
+}

src/cmd/run.go

@@ -0,0 +1,22 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var (
+	resyncInterval int
+	batchSize      int
+)
+
+var runCmd = &cobra.Command{
+	Use:   "run",
+	Short: "Commands for running a k8s controller",
+}
+
+func init() {
+	rootCmd.AddCommand(runCmd)
+
+	runCmd.PersistentFlags().IntVar(&resyncInterval, "resync", 24, "The amount (in hours) before a full resync of the kubernetes cluster happens with OpsLevel. [default: 24]")
+	runCmd.PersistentFlags().IntVar(&batchSize, "batch", 500, "The max amount of k8s resources to batch process with jq. Helps to speedup initial startup. [default: 500]")
+}