Merge branch 'main' into init-pkgs-version

src/api/v1alpha1/component.go

@@ -157,6 +157,16 @@ type ZarfChart struct {
 	ValuesFiles []string `json:"valuesFiles,omitempty"`
 	// [alpha] List of variables to set in the Helm chart.
 	Variables []ZarfChartVariable `json:"variables,omitempty"`
+	// Whether or not to validate the values.yaml schema, defaults to true. Necessary in the air-gap when the JSON Schema references resources on the internet.
+	SchemaValidation *bool `json:"schemaValidation,omitempty"`
+}
+
+// ShouldRunSchemaValidation returns if Helm schema validation should be run or not
+func (zc ZarfChart) ShouldRunSchemaValidation() bool {
+	if zc.SchemaValidation != nil {
+		return *zc.SchemaValidation
+	}
+	return true
 }
 
 // ZarfChartVariable represents a variable that can be set for a Helm chart overrides.

src/cmd/dev.go

@@ -23,6 +23,7 @@ import (
 	"github.com/zarf-dev/zarf/src/config"
 	"github.com/zarf-dev/zarf/src/config/lang"
 	"github.com/zarf-dev/zarf/src/pkg/lint"
+	"github.com/zarf-dev/zarf/src/pkg/logger"
 	"github.com/zarf-dev/zarf/src/pkg/message"
 	"github.com/zarf-dev/zarf/src/pkg/packager"
 	"github.com/zarf-dev/zarf/src/pkg/transform"
@@ -113,13 +114,17 @@ var devTransformGitLinksCmd = &cobra.Command{
 			return fmt.Errorf("unable to read the file %s: %w", fileName, err)
 		}
 
-		pkgConfig.InitOpts.GitServer.Address = host
+		gitServer := pkgConfig.InitOpts.GitServer
+		gitServer.Address = host
 
 		// Perform git url transformation via regex
 		text := string(content)
-		processedText := transform.MutateGitURLsInText(message.Warnf, pkgConfig.InitOpts.GitServer.Address, text, pkgConfig.InitOpts.GitServer.PushUsername)
+
+		// TODO(mkcp): Currently uses message for its log fn. Migrate to ctx and slog
+		processedText := transform.MutateGitURLsInText(message.Warnf, gitServer.Address, text, gitServer.PushUsername)
 
 		// Print the differences
+		// TODO(mkcp): Uses pterm to print text diffs. Decouple from pterm after we release logger.
 		dmp := diffmatchpatch.New()
 		diffs := dmp.DiffMain(text, processedText, true)
 		diffs = dmp.DiffCleanupSemantic(diffs)
@@ -160,6 +165,7 @@ var devSha256SumCmd = &cobra.Command{
 
 		if helpers.IsURL(fileName) {
 			message.Warn(lang.CmdDevSha256sumRemoteWarning)
+			logger.From(cmd.Context()).Warn("this is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link")
 
 			fileBase, err := helpers.ExtractBasePathFromURL(fileName)
 			if err != nil {
@@ -251,8 +257,14 @@ var devFindImagesCmd = &cobra.Command{
 		defer pkgClient.ClearTempPaths()
 
 		_, err = pkgClient.FindImages(cmd.Context())
+
 		var lintErr *lint.LintError
 		if errors.As(err, &lintErr) {
+			// HACK(mkcp): Re-initializing PTerm with a stderr writer isn't great, but it lets us render these lint
+			// tables below for backwards compatibility
+			if logger.Enabled(cmd.Context()) {
+				message.InitializePTerm(logger.DestinationDefault)
+			}
 			common.PrintFindings(lintErr)
 		}
 		if err != nil {
@@ -299,6 +311,11 @@ var devLintCmd = &cobra.Command{
 		err := lint.Validate(cmd.Context(), pkgConfig.CreateOpts.BaseDir, pkgConfig.CreateOpts.Flavor, pkgConfig.CreateOpts.SetVariables)
 		var lintErr *lint.LintError
 		if errors.As(err, &lintErr) {
+			// HACK(mkcp): Re-initializing PTerm with a stderr writer isn't great, but it lets us render these lint
+			// tables below for backwards compatibility
+			if logger.Enabled(cmd.Context()) {
+				message.InitializePTerm(logger.DestinationDefault)
+			}
 			common.PrintFindings(lintErr)
 			// Do not return an error if the findings are all warnings.
 			if lintErr.OnlyWarnings() {

src/cmd/initialize.go

@@ -37,6 +37,8 @@ var initCmd = &cobra.Command{
 	Long:    lang.CmdInitLong,
 	Example: lang.CmdInitExample,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		ctx := cmd.Context()
+
 		if err := validateInitFlags(); err != nil {
 			return fmt.Errorf("invalid command flags were provided: %w", err)
 		}
@@ -59,7 +61,7 @@ var initCmd = &cobra.Command{
 			return err
 		}
 
-		src, err := sources.New(&pkgConfig.PkgOpts)
+		src, err := sources.New(ctx, &pkgConfig.PkgOpts)
 		if err != nil {
 			return err
 		}
@@ -74,7 +76,7 @@ var initCmd = &cobra.Command{
 		}
 		defer pkgClient.ClearTempPaths()
 
-		err = pkgClient.Deploy(cmd.Context())
+		err = pkgClient.Deploy(ctx)
 		if err != nil {
 			return err
 		}
@@ -151,7 +153,7 @@ func downloadInitPackage(ctx context.Context, cacheDirectory string) (string, er
 
 	// If the user wants to download the init-package, download it
 	if confirmDownload {
-		remote, err := zoci.NewRemote(url, oci.PlatformForArch(config.GetArch()))
+		remote, err := zoci.NewRemote(ctx, url, oci.PlatformForArch(config.GetArch()))
 		if err != nil {
 			return "", err
 		}

src/cmd/tools/zarf.go

@@ -222,13 +222,14 @@ var downloadInitCmd = &cobra.Command{
 	Use:   "download-init",
 	Short: lang.CmdToolsDownloadInitShort,
 	RunE: func(cmd *cobra.Command, _ []string) error {
+		ctx := cmd.Context()
 		url := zoci.GetInitPackageURL(config.CLIVersion)
-		remote, err := zoci.NewRemote(url, oci.PlatformForArch(config.GetArch()))
+		remote, err := zoci.NewRemote(ctx, url, oci.PlatformForArch(config.GetArch()))
 		if err != nil {
 			return fmt.Errorf("unable to download the init package: %w", err)
 		}
 		source := &sources.OCISource{Remote: remote}
-		_, err = source.Collect(cmd.Context(), outputDirectory)
+		_, err = source.Collect(ctx, outputDirectory)
 		if err != nil {
 			return fmt.Errorf("unable to download the init package: %w", err)
 		}

src/internal/git/repository.go

@@ -18,6 +18,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 
+	"github.com/zarf-dev/zarf/src/pkg/logger"
 	"github.com/zarf-dev/zarf/src/pkg/message"
 	"github.com/zarf-dev/zarf/src/pkg/transform"
 	"github.com/zarf-dev/zarf/src/pkg/utils"
@@ -146,6 +147,7 @@ func (r *Repository) Path() string {
 
 // Push pushes the repository to the remote git server.
 func (r *Repository) Push(ctx context.Context, address, username, password string) error {
+	l := logger.From(ctx)
 	repo, err := git.PlainOpen(r.path)
 	if err != nil {
 		return fmt.Errorf("not a valid git repo or unable to open: %w", err)
@@ -195,10 +197,13 @@ func (r *Repository) Push(ctx context.Context, address, username, password strin
 	err = repo.FetchContext(ctx, fetchOptions)
 	if errors.Is(err, transport.ErrRepositoryNotFound) {
 		message.Debugf("Repo not yet available offline, skipping fetch...")
+		l.Debug("repo not yet available offline, skipping fetch")
 	} else if errors.Is(err, git.ErrForceNeeded) {
 		message.Debugf("Repo fetch requires force, skipping fetch...")
+		l.Debug("repo fetch requires force, skipping fetch")
 	} else if errors.Is(err, git.NoErrAlreadyUpToDate) {
 		message.Debugf("Repo already up-to-date, skipping fetch...")
+		l.Debug("repo already up-to-date, skipping fetch")
 	} else if err != nil {
 		return fmt.Errorf("unable to fetch the git repo prior to push: %w", err)
 	}
@@ -217,6 +222,7 @@ func (r *Repository) Push(ctx context.Context, address, username, password strin
 	})
 	if errors.Is(err, git.NoErrAlreadyUpToDate) {
 		message.Debug("Repo already up-to-date")
+		l.Debug("repo already up-to-date")
 	} else if errors.Is(err, plumbing.ErrObjectNotFound) {
 		return fmt.Errorf("unable to push repo due to likely shallow clone: %s", err.Error())
 	} else if err != nil {

src/internal/packager/helm/chart.go

@@ -9,9 +9,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/zarf-dev/zarf/src/pkg/logger"
 	"time"
 
+	"github.com/zarf-dev/zarf/src/pkg/logger"
+
 	"github.com/Masterminds/semver/v3"
 	"github.com/avast/retry-go/v4"
 	plutoversionsfile "github.com/fairwindsops/pluto/v5"
@@ -35,23 +36,26 @@ import (
 
 // InstallOrUpgradeChart performs a helm install of the given chart.
 func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings, string, error) {
-	fromMessage := h.chart.URL
-	if fromMessage == "" {
-		fromMessage = "Zarf-generated helm chart"
+	l := logger.From(ctx)
+	start := time.Now()
+	source := h.chart.URL
+	if source == "" {
+		source = "Zarf-generated"
 	}
-	spinner := message.NewProgressSpinner("Processing helm chart %s:%s from %s",
+	spinner := message.NewProgressSpinner("Processing helm chart %s:%s source: %s",
 		h.chart.Name,
 		h.chart.Version,
-		fromMessage)
+		source)
 	defer spinner.Stop()
+	l.Info("processing helm chart", "name", h.chart.Name, "version", h.chart.Version, "source", source)
 
 	// If no release name is specified, use the chart name.
 	if h.chart.ReleaseName == "" {
 		h.chart.ReleaseName = h.chart.Name
 	}
 
 	// Setup K8s connection.
-	err := h.createActionConfig(h.chart.Namespace, spinner)
+	err := h.createActionConfig(ctx, h.chart.Namespace, spinner)
 	if err != nil {
 		return nil, "", fmt.Errorf("unable to initialize the K8s client: %w", err)
 	}
@@ -73,15 +77,18 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings,
 		releases, histErr := histClient.Run(h.chart.ReleaseName)
 
 		spinner.Updatef("Checking for existing helm deployment")
+		l.Debug("checking for existing helm deployment")
 
 		if errors.Is(histErr, driver.ErrReleaseNotFound) {
 			// No prior release, try to install it.
 			spinner.Updatef("Attempting chart installation")
+			l.Info("performing Helm install", "chart", h.chart.Name)
 
 			release, err = h.installChart(helmCtx, postRender)
 		} else if histErr == nil && len(releases) > 0 {
 			// Otherwise, there is a prior release so upgrade it.
 			spinner.Updatef("Attempting chart upgrade")
+			l.Info("performing Helm upgrade", "chart", h.chart.Name)
 
 			lastRelease := releases[len(releases)-1]
 
@@ -118,6 +125,7 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings,
 
 		// Attempt to rollback on a failed upgrade.
 		spinner.Updatef("Performing chart rollback")
+		l.Info("performing Helm rollback", "chart", h.chart.Name)
 		err = h.rollbackChart(h.chart.ReleaseName, previouslyDeployedVersion)
 		if err != nil {
 			return nil, "", fmt.Errorf("%w: unable to rollback: %w", installErr, err)
@@ -137,11 +145,13 @@ func (h *Helm) InstallOrUpgradeChart(ctx context.Context) (types.ConnectStrings,
 	if !h.chart.NoWait {
 		// Ensure we don't go past the timeout by using a context initialized with the helm timeout
 		spinner.Updatef("Running health checks")
+		l.Info("running health checks", "chart", h.chart.Name)
 		if err := healthchecks.WaitForReadyRuntime(helmCtx, h.cluster.Watcher, runtimeObjs); err != nil {
 			return nil, "", err
 		}
 	}
 	spinner.Success()
+	l.Debug("done processing helm chart", "name", h.chart.Name, "duration", time.Since(start))
 
 	// return any collected connect strings for zarf connect.
 	return postRender.connectStrings, h.chart.ReleaseName, nil
@@ -152,7 +162,7 @@ func (h *Helm) TemplateChart(ctx context.Context) (manifest string, chartValues
 	spinner := message.NewProgressSpinner("Templating helm chart %s", h.chart.Name)
 	defer spinner.Stop()
 
-	err = h.createActionConfig(h.chart.Namespace, spinner)
+	err = h.createActionConfig(ctx, h.chart.Namespace, spinner)
 
 	// Setup K8s connection.
 	if err != nil {
@@ -216,7 +226,7 @@ func (h *Helm) TemplateChart(ctx context.Context) (manifest string, chartValues
 // RemoveChart removes a chart from the cluster.
 func (h *Helm) RemoveChart(ctx context.Context, namespace string, name string, spinner *message.Spinner) error {
 	// Establish a new actionConfig for the namespace.
-	_ = h.createActionConfig(namespace, spinner)
+	_ = h.createActionConfig(ctx, namespace, spinner)
 	// Perform the uninstall.
 	response, err := h.uninstallChart(name)
 	message.Debug(response)
@@ -230,7 +240,7 @@ func (h *Helm) UpdateReleaseValues(ctx context.Context, updatedValues map[string
 	spinner := message.NewProgressSpinner("Updating values for helm release %s", h.chart.ReleaseName)
 	defer spinner.Stop()
 
-	err := h.createActionConfig(h.chart.Namespace, spinner)
+	err := h.createActionConfig(ctx, h.chart.Namespace, spinner)
 	if err != nil {
 		return fmt.Errorf("unable to initialize the K8s client: %w", err)
 	}
@@ -296,6 +306,8 @@ func (h *Helm) installChart(ctx context.Context, postRender *renderer) (*release
 	// Must be unique per-namespace and < 53 characters. @todo: restrict helm loadedChart name to this.
 	client.ReleaseName = h.chart.ReleaseName
 
+	client.SkipSchemaValidation = !h.chart.ShouldRunSchemaValidation()
+
 	// Namespace must be specified.
 	client.Namespace = h.chart.Namespace
 
@@ -329,6 +341,8 @@ func (h *Helm) upgradeChart(ctx context.Context, lastRelease *release.Release, p
 
 	client.SkipCRDs = true
 
+	client.SkipSchemaValidation = !h.chart.ShouldRunSchemaValidation()
+
 	// Namespace must be specified.
 	client.Namespace = h.chart.Namespace
 

src/internal/packager/helm/destroy.go

@@ -26,7 +26,7 @@ func Destroy(ctx context.Context, purgeAllZarfInstallations bool) {
 	h := Helm{}
 
 	// Initially load the actionConfig without a namespace
-	err := h.createActionConfig("", spinner)
+	err := h.createActionConfig(ctx, "", spinner)
 	if err != nil {
 		// Don't fatal since this is a removal action
 		spinner.Errorf(err, "Unable to initialize the K8s client")

src/internal/packager/helm/post-render.go

@@ -16,6 +16,7 @@ import (
 	"github.com/defenseunicorns/pkg/helpers/v2"
 	"github.com/zarf-dev/zarf/src/config"
 	"github.com/zarf-dev/zarf/src/pkg/cluster"
+	"github.com/zarf-dev/zarf/src/pkg/logger"
 	"github.com/zarf-dev/zarf/src/pkg/message"
 	"github.com/zarf-dev/zarf/src/pkg/utils"
 	"github.com/zarf-dev/zarf/src/types"
@@ -117,6 +118,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) {
 }
 
 func (r *renderer) adoptAndUpdateNamespaces(ctx context.Context) error {
+	l := logger.From(ctx)
 	c := r.cluster
 	namespaceList, err := r.cluster.Clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
 	if err != nil {
@@ -142,6 +144,7 @@ func (r *renderer) adoptAndUpdateNamespaces(ctx context.Context) error {
 			// https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/#initial-namespaces
 			if slices.Contains([]string{"default", "kube-node-lease", "kube-public", "kube-system"}, name) {
 				message.Warnf("Refusing to adopt the initial namespace: %s", name)
+				l.Warn("refusing to adopt initial namespace", "name", name)
 			} else {
 				// This is an existing namespace to adopt
 				_, err := c.Clientset.CoreV1().Namespaces().Update(ctx, namespace, metav1.UpdateOptions{})
@@ -181,6 +184,7 @@ func (r *renderer) adoptAndUpdateNamespaces(ctx context.Context) error {
 			}()
 			if err != nil {
 				message.WarnErrf(err, "Problem creating registry secret for the %s namespace", name)
+				l.Warn("problem creating registry secret", "namespace", name, "error", err.Error())
 			}
 
 			// Create or update the zarf git server secret
@@ -201,13 +205,15 @@ func (r *renderer) adoptAndUpdateNamespaces(ctx context.Context) error {
 			}()
 			if err != nil {
 				message.WarnErrf(err, "Problem creating git server secret for the %s namespace", name)
+				l.Warn("problem creating git server secret", "namespace", name, "error", err.Error())
 			}
 		}
 	}
 	return nil
 }
 
 func (r *renderer) editHelmResources(ctx context.Context, resources []releaseutil.Manifest, finalManifestsOutput *bytes.Buffer) error {
+	l := logger.From(ctx)
 	dc, err := dynamic.NewForConfig(r.cluster.RestConfig)
 	if err != nil {
 		return err
@@ -231,8 +237,10 @@ func (r *renderer) editHelmResources(ctx context.Context, resources []releaseuti
 			// parse the namespace resource so it can be applied out-of-band by zarf instead of helm to avoid helm ns shenanigans
 			if err := runtime.DefaultUnstructuredConverter.FromUnstructured(rawData.UnstructuredContent(), namespace); err != nil {
 				message.WarnErrf(err, "could not parse namespace %s", rawData.GetName())
+				l.Warn("failed to parse namespace", "name", rawData.GetName(), "error", err)
 			} else {
 				message.Debugf("Matched helm namespace %s for zarf annotation", namespace.Name)
+				l.Debug("matched helm namespace for zarf annotation", "name", namespace.Name)
 				namespace.Labels = cluster.AdoptZarfManagedLabels(namespace.Labels)
 				// Add it to the stack
 				r.namespaces[namespace.Name] = namespace
@@ -253,6 +261,7 @@ func (r *renderer) editHelmResources(ctx context.Context, resources []releaseuti
 			if key, keyExists := labels[cluster.ZarfConnectLabelName]; keyExists {
 				// If there is a zarf-connect label
 				message.Debugf("Match helm service %s for zarf connection %s", rawData.GetName(), key)
+				l.Debug("match helm service for zarf connection", "service", rawData.GetName(), "connection-key", key)
 
 				// Add the connectString for processing later in the deployment
 				r.connectStrings[key] = types.ConnectString{