From 0e1c13ec638eededd99ef63613fcb07951c11117 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 23 Jan 2024 13:52:14 +0000 Subject: [PATCH 001/113] WIP, OCI tests passing --- src/cmd/initialize.go | 2 +- src/cmd/tools/zarf.go | 2 +- src/internal/packager/images/pull.go | 1 + src/pkg/oci/common.go | 45 +++-- src/pkg/oci/copier.go | 9 +- src/pkg/oci/fetch.go | 10 +- src/pkg/oci/interface.go | 11 ++ src/pkg/oci/manifest.go | 25 +-- src/pkg/oci/progress.go | 2 +- src/pkg/oci/pull.go | 194 ++++++++++----------- src/pkg/oci/utils.go | 3 +- src/pkg/packager/composer/oci.go | 2 +- src/pkg/packager/create_stages.go | 4 +- src/pkg/packager/publish.go | 4 +- src/pkg/packager/sources/new.go | 2 +- src/pkg/utils/transport.go | 12 +- src/pkg/zarfoci/manifest.go | 16 ++ src/pkg/zarfoci/pull.go | 100 +++++++++++ src/test/e2e/50_oci_publish_deploy_test.go | 5 +- 19 files changed, 290 insertions(+), 159 deletions(-) create mode 100644 src/pkg/oci/interface.go create mode 100644 src/pkg/zarfoci/manifest.go create mode 100644 src/pkg/zarfoci/pull.go diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 799896b066..c1dcd00681 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -138,7 +138,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { // If the user wants to download the init-package, download it if confirmDownload { - remote, err := oci.NewOrasRemote(url, oci.WithArch(config.GetArch())) + remote, err := oci.NewOrasRemote(url, message.Infof, oci.WithArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return "", err } diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index d835674ed6..6da3c64fa7 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -183,7 +183,7 @@ var downloadInitCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { url := oci.GetInitPackageURL(config.CLIVersion) - remote, err := oci.NewOrasRemote(url, oci.WithArch(config.GetArch())) + remote, err := oci.NewOrasRemote(url, message.Infof, oci.WithArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index ed115a1d65..692655bc0f 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -74,6 +74,7 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { spinner.Updatef("Fetching image metadata (0 of %d)", len(i.ImageList)) // Spawn a goroutine for each image to load its metadata + // I would like to understand this code better for _, refInfo := range i.ImageList { // Create a closure so that we can pass the src into the goroutine refInfo := refInfo diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index c228ec0675..fe17993be8 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -10,8 +10,6 @@ import ( "net/http" "strings" - zarfconfig "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/docker/cli/cli/config" @@ -34,14 +32,19 @@ const ( MultiOS = "multi" ) +// log is a function that logs a message. +type log func(string, ...any) + // OrasRemote is a wrapper around the Oras remote repository that includes a progress bar for interactive feedback. type OrasRemote struct { repo *remote.Repository - root *ZarfOCIManifest + root *OCIManifest ctx context.Context Transport *utils.Transport CopyOpts oras.CopyOptions targetPlatform *ocispec.Platform + userAgent string + log log } // Modifier is a function that modifies an OrasRemote @@ -54,6 +57,15 @@ func WithContext(ctx context.Context) Modifier { } } +func WithInsecure(insecure bool) Modifier { + return func(o *OrasRemote) { + plainHTTPMod := WithPlainHTTP(insecure) + plainHTTPMod(o) + insecureTLSMod := WithInsecureSkipVerify(insecure) + insecureTLSMod(o) + } +} + // WithCopyOpts sets the copy options for the remote func WithCopyOpts(opts oras.CopyOptions) Modifier { return func(o *OrasRemote) { @@ -98,15 +110,23 @@ func WithArch(arch string) Modifier { }) } +// WithArch sets the target architecture for the remote +func WithUserAgent(userAgent string) Modifier { + return func(o *OrasRemote) { + o.userAgent = userAgent + } +} + // NewOrasRemote returns an oras remote repository client and context for the given url. // // Registry auth is handled by the Docker CLI's credential store and checked before returning the client -func NewOrasRemote(url string, mods ...Modifier) (*OrasRemote, error) { +func NewOrasRemote(url string, logger log, mods ...Modifier) (*OrasRemote, error) { ref, err := registry.ParseReference(strings.TrimPrefix(url, helpers.OCIURLPrefix)) if err != nil { return nil, fmt.Errorf("failed to parse OCI reference %q: %w", url, err) } o := &OrasRemote{} + o.log = logger if err := o.setRepository(ref); err != nil { return nil, err @@ -117,15 +137,6 @@ func NewOrasRemote(url string, mods ...Modifier) (*OrasRemote, error) { copyOpts.PostCopy = o.printLayerCopied o.CopyOpts = copyOpts - // right now --insecure is overloaded to mean both plain HTTP and insecure TLS - // putting this here as the "default" for the remote - // but can be overridden by a provided modifier - insecureMod := WithInsecureSkipVerify(zarfconfig.CommonOptions.Insecure) - insecureMod(o) - - httpMod := WithPlainHTTP(zarfconfig.CommonOptions.Insecure) - httpMod(o) - for _, mod := range mods { mod(o) } @@ -186,15 +197,17 @@ func (o *OrasRemote) createAuthClient(ref registry.Reference) (*auth.Client, err Transport: o.Transport, }, } - client.SetUserAgent("zarf/" + zarfconfig.CLIVersion) + if o.userAgent != "" { + client.SetUserAgent(o.userAgent) + } - message.Debugf("Loading docker config file from default config location: %s for %s", config.Dir(), ref) + o.log("Loading docker config file from default config location: %s for %s", config.Dir(), ref) cfg, err := config.Load(config.Dir()) if err != nil { return nil, err } if !cfg.ContainsAuth() { - message.Debug("no docker config file found, run 'zarf tools registry login --help'") + o.log("no docker config file found, run 'zarf tools registry login --help'") return client, nil } diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index aa901c8284..f74d0cd5ab 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -17,7 +17,8 @@ import ( ) // CopyPackage copies a package from one OCI registry to another -func CopyPackage(ctx context.Context, src *OrasRemote, dst *OrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { +func CopyPackage(ctx context.Context, src *OrasRemote, dst *OrasRemote, + include func(d ocispec.Descriptor) bool, concurrency int) error { // create a new semaphore to limit concurrency sem := semaphore.NewWeighted(int64(concurrency)) @@ -49,7 +50,7 @@ func CopyPackage(ctx context.Context, src *OrasRemote, dst *OrasRemote, include start := time.Now() for idx, layer := range layers { - message.Debug("Copying layer:", message.JSONValue(layer)) + src.log("Copying layer:", message.JSONValue(layer)) if err := sem.Acquire(ctx, 1); err != nil { return err } @@ -60,7 +61,7 @@ func CopyPackage(ctx context.Context, src *OrasRemote, dst *OrasRemote, include return err } if exists { - message.Debug("Layer already exists in destination, skipping") + src.log("Layer already exists in destination, skipping") progressBar.UpdateTitle(fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) progressBar.Add(int(layer.Size)) sem.Release(1) @@ -112,7 +113,7 @@ func CopyPackage(ctx context.Context, src *OrasRemote, dst *OrasRemote, include } duration := time.Since(start) - message.Debug("Copied", src.repo.Reference, "to", dst.repo.Reference, "with a concurrency of", concurrency, "and took", duration) + src.log("Copied", src.repo.Reference, "to", dst.repo.Reference, "with a concurrency of", concurrency, "and took", duration) return nil } diff --git a/src/pkg/oci/fetch.go b/src/pkg/oci/fetch.go index 1b95e61efd..7ca32cd0a3 100644 --- a/src/pkg/oci/fetch.go +++ b/src/pkg/oci/fetch.go @@ -42,7 +42,7 @@ func (o *OrasRemote) ResolveRoot() (ocispec.Descriptor, error) { } // FetchRoot fetches the root manifest from the remote repository. -func (o *OrasRemote) FetchRoot() (*ZarfOCIManifest, error) { +func (o *OrasRemote) FetchRoot() (*OCIManifest, error) { if o.root != nil { return o.root, nil } @@ -62,8 +62,8 @@ func (o *OrasRemote) FetchRoot() (*ZarfOCIManifest, error) { } // FetchManifest fetches the manifest with the given descriptor from the remote repository. -func (o *OrasRemote) FetchManifest(desc ocispec.Descriptor) (manifest *ZarfOCIManifest, err error) { - return FetchUnmarshal[*ZarfOCIManifest](o.FetchLayer, json.Unmarshal, desc) +func (o *OrasRemote) FetchManifest(desc ocispec.Descriptor) (manifest *OCIManifest, err error) { + return FetchUnmarshal[*OCIManifest](o.FetchLayer, json.Unmarshal, desc) } // FetchLayer fetches the layer with the given descriptor from the remote repository. @@ -90,7 +90,7 @@ func (o *OrasRemote) FetchImagesIndex() (index *ocispec.Index, err error) { } // FetchJSONFile fetches the given JSON file from the remote repository. -func FetchJSONFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *ZarfOCIManifest, path string) (result T, err error) { +func FetchJSONFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *OCIManifest, path string) (result T, err error) { descriptor := manifest.Locate(path) if IsEmptyDescriptor(descriptor) { return result, fmt.Errorf("unable to find %s in the manifest", path) @@ -99,7 +99,7 @@ func FetchJSONFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, e } // FetchYAMLFile fetches the given YAML file from the remote repository. -func FetchYAMLFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *ZarfOCIManifest, path string) (result T, err error) { +func FetchYAMLFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *OCIManifest, path string) (result T, err error) { descriptor := manifest.Locate(path) if IsEmptyDescriptor(descriptor) { return result, fmt.Errorf("unable to find %s in the manifest", path) diff --git a/src/pkg/oci/interface.go b/src/pkg/oci/interface.go new file mode 100644 index 0000000000..4d67fcb102 --- /dev/null +++ b/src/pkg/oci/interface.go @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package oci + +type OrasObj interface { + Pull() + Push() + New() +} diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index dc23f8aee8..ca8c559443 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -12,30 +12,21 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -var ( - // ZarfPackageIndexPath is the path to the index.json file in the OCI package. - ZarfPackageIndexPath = filepath.Join("images", "index.json") - // ZarfPackageLayoutPath is the path to the oci-layout file in the OCI package. - ZarfPackageLayoutPath = filepath.Join("images", "oci-layout") - // ZarfPackageImagesBlobsDir is the path to the directory containing the image blobs in the OCI package. - ZarfPackageImagesBlobsDir = filepath.Join("images", "blobs", "sha256") -) - -// ZarfOCIManifest is a wrapper around the OCI manifest +// OCIManifest is a wrapper around the OCI manifest // // it includes the path to the index.json, oci-layout, and image blobs. // as well as a few helper functions for locating layers and calculating the size of the layers. -type ZarfOCIManifest struct { +type OCIManifest struct { ocispec.Manifest } -// NewZarfOCIManifest returns a new ZarfOCIManifest. -func NewZarfOCIManifest(manifest *ocispec.Manifest) *ZarfOCIManifest { - return &ZarfOCIManifest{*manifest} +// New returns a new OCIManifest +func New(manifest *ocispec.Manifest) *OCIManifest { + return &OCIManifest{*manifest} } // Locate returns the descriptor for the first layer with the given path or digest. -func (m *ZarfOCIManifest) Locate(pathOrDigest string) ocispec.Descriptor { +func (m *OCIManifest) Locate(pathOrDigest string) ocispec.Descriptor { return helpers.Find(m.Layers, func(layer ocispec.Descriptor) bool { // Convert from the OS path separator to the standard '/' for Windows support return layer.Annotations[ocispec.AnnotationTitle] == filepath.ToSlash(pathOrDigest) || layer.Digest.Encoded() == pathOrDigest @@ -43,7 +34,7 @@ func (m *ZarfOCIManifest) Locate(pathOrDigest string) ocispec.Descriptor { } // SumLayersSize returns the sum of the size of all the layers in the manifest. -func (m *ZarfOCIManifest) SumLayersSize() int64 { +func (m *OCIManifest) SumLayersSize() int64 { var sum int64 for _, layer := range m.Layers { sum += layer.Size @@ -52,6 +43,6 @@ func (m *ZarfOCIManifest) SumLayersSize() int64 { } // MarshalJSON returns the JSON encoding of the manifest. -func (m *ZarfOCIManifest) MarshalJSON() ([]byte, error) { +func (m *OCIManifest) MarshalJSON() ([]byte, error) { return json.Marshal(m.Manifest) } diff --git a/src/pkg/oci/progress.go b/src/pkg/oci/progress.go index 2c0ad4769b..7c262000f7 100644 --- a/src/pkg/oci/progress.go +++ b/src/pkg/oci/progress.go @@ -31,6 +31,6 @@ func (o *OrasRemote) printLayer(desc ocispec.Descriptor, suffix string) error { } else { layerInfo = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) } - message.Debugf("%s (%s)", layerInfo, suffix) + o.log("%s (%s)", layerInfo, suffix) return nil } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 8bebcee533..572a4d26ee 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -14,7 +14,6 @@ import ( "slices" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -25,6 +24,15 @@ import ( "oras.land/oras-go/v2/content/file" ) +var ( + // ZarfPackageIndexPath is the path to the index.json file in the OCI package. + ZarfPackageIndexPath = filepath.Join("images", "index.json") + // ZarfPackageLayoutPath is the path to the oci-layout file in the OCI package. + ZarfPackageLayoutPath = filepath.Join("images", "oci-layout") + // ZarfPackageImagesBlobsDir is the path to the directory containing the image blobs in the OCI package. + ZarfPackageImagesBlobsDir = filepath.Join("images", "blobs", "sha256") +) + var ( // PackageAlwaysPull is a list of paths that will always be pulled from the remote repository. PackageAlwaysPull = []string{layout.ZarfYAML, layout.Checksums, layout.Signature} @@ -59,105 +67,6 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi return actual == desc.Digest.Encoded() } -// LayersFromPaths returns the descriptors for the given paths from the root manifest. -func (o *OrasRemote) LayersFromPaths(requestedPaths []string) (layers []ocispec.Descriptor, err error) { - manifest, err := o.FetchRoot() - if err != nil { - return nil, err - } - for _, path := range requestedPaths { - layer := manifest.Locate(path) - if IsEmptyDescriptor(layer) { - return nil, fmt.Errorf("path %s does not exist in this package", path) - } - layers = append(layers, layer) - } - return layers, nil -} - -// LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. -// -// It also retrieves the descriptors for all image layers that are required by the components. -// -// It also respects the `required` flag on components, and will retrieve all necessary layers for required components. -func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) (layers []ocispec.Descriptor, err error) { - root, err := o.FetchRoot() - if err != nil { - return nil, err - } - - pkg, err := o.FetchZarfYAML() - if err != nil { - return nil, err - } - images := map[string]bool{} - tarballFormat := "%s.tar" - for _, name := range requestedComponents { - component := helpers.Find(pkg.Components, func(component types.ZarfComponent) bool { - return component.Name == name - }) - if component.Name == "" { - return nil, fmt.Errorf("component %s does not exist in this package", name) - } - } - for _, component := range pkg.Components { - // If we requested this component, or it is required, we need to pull its images and tarball - if slices.Contains(requestedComponents, component.Name) || component.Required { - for _, image := range component.Images { - images[image] = true - } - layers = append(layers, root.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf(tarballFormat, component.Name)))) - } - } - // Append the sboms.tar layer if it exists - // - // Since sboms.tar is not a heavy addition 99% of the time, we'll just always pull it - sbomsDescriptor := root.Locate(layout.SBOMTar) - if !IsEmptyDescriptor(sbomsDescriptor) { - layers = append(layers, sbomsDescriptor) - } - if len(images) > 0 { - // Add the image index and the oci-layout layers - layers = append(layers, root.Locate(ZarfPackageIndexPath), root.Locate(ZarfPackageLayoutPath)) - index, err := o.FetchImagesIndex() - if err != nil { - return nil, err - } - for image := range images { - // use docker's transform lib to parse the image ref - // this properly mirrors the logic within create - refInfo, err := transform.ParseImageRef(image) - if err != nil { - return nil, fmt.Errorf("failed to parse image ref %q: %w", image, err) - } - - manifestDescriptor := helpers.Find(index.Manifests, func(layer ocispec.Descriptor) bool { - return layer.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Reference || - // A backwards compatibility shim for older Zarf versions that would leave docker.io off of image annotations - (layer.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Path+refInfo.TagOrDigest && refInfo.Host == "docker.io") - }) - - // even though these are technically image manifests, we store them as Zarf blobs - manifestDescriptor.MediaType = ZarfLayerMediaTypeBlob - - manifest, err := o.FetchManifest(manifestDescriptor) - if err != nil { - return nil, err - } - // Add the manifest and the manifest config layers - layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifestDescriptor.Digest.Encoded()))) - layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifest.Config.Digest.Encoded()))) - - // Add all the layers from the manifest - for _, layer := range manifest.Layers { - layerPath := filepath.Join(ZarfPackageImagesBlobsDir, layer.Digest.Encoded()) - layers = append(layers, root.Locate(layerPath)) - } - } - } - return layers, nil -} - // PullPackage pulls the package from the remote repository and saves it to the given path. // // layersToPull is an optional parameter that allows the caller to specify which layers to pull. @@ -168,7 +77,7 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) // - zarf.yaml.sig func (o *OrasRemote) PullPackage(destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { isPartialPull := len(layersToPull) > 0 - message.Debug("Pulling", o.repo.Reference) + o.log("Pulling", o.repo.Reference) manifest, err := o.FetchRoot() if err != nil { @@ -309,3 +218,86 @@ func (o *OrasRemote) PullPackageMetadata(destinationDir string) ([]ocispec.Descr func (o *OrasRemote) PullPackageSBOM(destinationDir string) ([]ocispec.Descriptor, error) { return o.PullPackagePaths([]string{layout.SBOMTar}, destinationDir) } + +// LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. +// +// It also retrieves the descriptors for all image layers that are required by the components. +// +// It also respects the `required` flag on components, and will retrieve all necessary layers for required components. +func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) (layers []ocispec.Descriptor, err error) { + root, err := o.FetchRoot() + if err != nil { + return nil, err + } + + pkg, err := o.FetchZarfYAML() + if err != nil { + return nil, err + } + images := map[string]bool{} + tarballFormat := "%s.tar" + for _, name := range requestedComponents { + component := helpers.Find(pkg.Components, func(component types.ZarfComponent) bool { + return component.Name == name + }) + if component.Name == "" { + return nil, fmt.Errorf("component %s does not exist in this package", name) + } + } + for _, component := range pkg.Components { + // If we requested this component, or it is required, we need to pull its images and tarball + if slices.Contains(requestedComponents, component.Name) || component.Required { + for _, image := range component.Images { + images[image] = true + } + layers = append(layers, root.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf(tarballFormat, component.Name)))) + } + } + // Append the sboms.tar layer if it exists + // + // Since sboms.tar is not a heavy addition 99% of the time, we'll just always pull it + sbomsDescriptor := root.Locate(layout.SBOMTar) + if !IsEmptyDescriptor(sbomsDescriptor) { + layers = append(layers, sbomsDescriptor) + } + if len(images) > 0 { + // Add the image index and the oci-layout layers + layers = append(layers, root.Locate(ZarfPackageIndexPath), root.Locate(ZarfPackageLayoutPath)) + index, err := o.FetchImagesIndex() + if err != nil { + return nil, err + } + for image := range images { + // use docker's transform lib to parse the image ref + // this properly mirrors the logic within create + refInfo, err := transform.ParseImageRef(image) + if err != nil { + return nil, fmt.Errorf("failed to parse image ref %q: %w", image, err) + } + + manifestDescriptor := helpers.Find(index.Manifests, func(layer ocispec.Descriptor) bool { + return layer.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Reference || + // A backwards compatibility shim for older Zarf versions that would leave docker.io off of image annotations + (layer.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Path+refInfo.TagOrDigest && refInfo.Host == "docker.io") + }) + + // even though these are technically image manifests, we store them as Zarf blobs + manifestDescriptor.MediaType = ZarfLayerMediaTypeBlob + + manifest, err := o.FetchManifest(manifestDescriptor) + if err != nil { + return nil, err + } + // Add the manifest and the manifest config layers + layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifestDescriptor.Digest.Encoded()))) + layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifest.Config.Digest.Encoded()))) + + // Add all the layers from the manifest + for _, layer := range manifest.Layers { + layerPath := filepath.Join(ZarfPackageImagesBlobsDir, layer.Digest.Encoded()) + layers = append(layers, root.Locate(layerPath)) + } + } + } + return layers, nil +} diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index 149d6d20b7..73019ba46d 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -9,7 +9,6 @@ import ( "fmt" "strings" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -35,7 +34,7 @@ func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata raw = fmt.Sprintf("%s-%s", raw, build.Flavor) } - message.Debug("Raw OCI reference from metadata:", raw) + //o.log("Raw OCI reference from metadata:", raw) ref, err := registry.ParseReference(raw) if err != nil { diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 0477d04b97..ba57996727 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -27,7 +27,7 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { return ic.remote, nil } var err error - ic.remote, err = oci.NewOrasRemote(url, oci.WithSkeletonArch()) + ic.remote, err = oci.NewOrasRemote(url, message.Infof, oci.WithSkeletonArch(), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return nil, err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 2ae7f06ad9..5b0de8bf59 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -250,7 +250,7 @@ func (p *Packager) output() error { if err != nil { return err } - remote, err := oci.NewOrasRemote(ref) + remote, err := oci.NewOrasRemote(ref, message.Infof, oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return err } @@ -652,7 +652,7 @@ func (p *Packager) loadDifferentialData() error { // Load the package spec of the package we're using as a 'reference' for the differential build if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := oci.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) + remote, err := oci.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, message.Infof, oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 6022eeada2..2b359d881d 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -37,7 +37,7 @@ func (p *Packager) Publish() (err error) { p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName arch := config.GetArch() - dstRemote, err := oci.NewOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.WithArch(arch)) + dstRemote, err := oci.NewOrasRemote(p.cfg.PublishOpts.PackageDestination, message.Infof, oci.WithArch(arch), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return err } @@ -112,7 +112,7 @@ func (p *Packager) Publish() (err error) { return err } - remote, err := oci.NewOrasRemote(ref) + remote, err := oci.NewOrasRemote(ref, message.Infof, oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return err } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 9cd8c07c1a..a9c7256c56 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -67,7 +67,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } arch := config.GetArch() - remote, err := oci.NewOrasRemote(pkgSrc, oci.WithArch(arch)) + remote, err := oci.NewOrasRemote(pkgSrc, message.Infof, oci.WithArch(arch), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return nil, err } diff --git a/src/pkg/utils/transport.go b/src/pkg/utils/transport.go index 437c1918cb..3d31d819d6 100644 --- a/src/pkg/utils/transport.go +++ b/src/pkg/utils/transport.go @@ -9,19 +9,25 @@ import ( "net/http" "time" - "github.com/defenseunicorns/zarf/src/pkg/message" "oras.land/oras-go/v2/registry/remote/retry" ) +type progressBar interface { + Add(int) + Write([]byte) (n int, err error) + Stop() + Successf(format string, args ...interface{}) +} + // Transport is an http.RoundTripper that keeps track of the in-flight // request and add hooks to report upload progress. type Transport struct { Base http.RoundTripper - ProgressBar *message.ProgressBar + ProgressBar progressBar } // NewTransport returns a custom transport that tracks an http.RoundTripper and a message.ProgressBar. -func NewTransport(base http.RoundTripper, bar *message.ProgressBar) *Transport { +func NewTransport(base http.RoundTripper, bar progressBar) *Transport { return &Transport{ Base: base, ProgressBar: bar, diff --git a/src/pkg/zarfoci/manifest.go b/src/pkg/zarfoci/manifest.go new file mode 100644 index 0000000000..53a8dc3a9a --- /dev/null +++ b/src/pkg/zarfoci/manifest.go @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package oci + +import "path/filepath" + +var ( + // ZarfPackageIndexPath is the path to the index.json file in the OCI package. + ZarfPackageIndexPath = filepath.Join("images", "index.json") + // ZarfPackageLayoutPath is the path to the oci-layout file in the OCI package. + ZarfPackageLayoutPath = filepath.Join("images", "oci-layout") + // ZarfPackageImagesBlobsDir is the path to the directory containing the image blobs in the OCI package. + ZarfPackageImagesBlobsDir = filepath.Join("images", "blobs", "sha256") +) diff --git a/src/pkg/zarfoci/pull.go b/src/pkg/zarfoci/pull.go new file mode 100644 index 0000000000..7ec4b89149 --- /dev/null +++ b/src/pkg/zarfoci/pull.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package oci + +import ( + "fmt" + "path/filepath" + "slices" + + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. +// It also retrieves the descriptors for all image layers that are required by the components. +// +// It also respects the `required` flag on components, and will retrieve all necessary layers for required components. +func LayersFromRequestedComponents(o *oci.OrasRemote, requestedComponents []string) (layers []ocispec.Descriptor, err error) { + root, err := o.FetchRoot() + if err != nil { + return nil, err + } + + pkg, err := o.FetchZarfYAML() + if err != nil { + return nil, err + } + images := map[string]bool{} + tarballFormat := "%s.tar" + for _, name := range requestedComponents { + component := helpers.Find(pkg.Components, func(component types.ZarfComponent) bool { + return component.Name == name + }) + if component.Name == "" { + return nil, fmt.Errorf("component %s does not exist in this package", name) + } + } + for _, component := range pkg.Components { + // If we requested this component, or it is required, we need to pull its images and tarball + if slices.Contains(requestedComponents, component.Name) || component.Required { + for _, image := range component.Images { + images[image] = true + } + layers = append(layers, root.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf(tarballFormat, component.Name)))) + } + } + // Append the sboms.tar layer if it exists + // + // Since sboms.tar is not a heavy addition 99% of the time, we'll just always pull it + sbomsDescriptor := root.Locate(layout.SBOMTar) + if !oci.IsEmptyDescriptor(sbomsDescriptor) { + layers = append(layers, sbomsDescriptor) + } + if len(images) > 0 { + // Add the image index and the oci-layout layers + layers = append(layers, root.Locate(ZarfPackageIndexPath), root.Locate(ZarfPackageLayoutPath)) + index, err := o.FetchImagesIndex() + if err != nil { + return nil, err + } + for image := range images { + // use docker's transform lib to parse the image ref + // this properly mirrors the logic within create + refInfo, err := transform.ParseImageRef(image) + if err != nil { + return nil, fmt.Errorf("failed to parse image ref %q: %w", image, err) + } + + manifestDescriptor := helpers.Find(index.Manifests, func(layer ocispec.Descriptor) bool { + return layer.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Reference || + // A backwards compatibility shim for older Zarf versions that would leave docker.io off of image annotations + (layer.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Path+refInfo.TagOrDigest && refInfo.Host == "docker.io") + }) + + // even though these are technically image manifests, we store them as Zarf blobs + manifestDescriptor.MediaType = oci.ZarfLayerMediaTypeBlob + + manifest, err := o.FetchManifest(manifestDescriptor) + if err != nil { + return nil, err + } + // Add the manifest and the manifest config layers + layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifestDescriptor.Digest.Encoded()))) + layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifest.Config.Digest.Encoded()))) + + // Add all the layers from the manifest + for _, layer := range manifest.Layers { + layerPath := filepath.Join(ZarfPackageImagesBlobsDir, layer.Digest.Encoded()) + layers = append(layers, root.Locate(layerPath)) + } + } + } + return layers, nil +} diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 55a038715b..d32950fbfd 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -128,10 +129,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - src, err := oci.NewOrasRemote(ref, oci.WithPlainHTTP(true), oci.WithArch(e2e.Arch)) + src, err := oci.NewOrasRemote(ref, message.Infof, oci.WithArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) - dst, err := oci.NewOrasRemote(dstRef, oci.WithPlainHTTP(true), oci.WithArch(e2e.Arch)) + dst, err := oci.NewOrasRemote(dstRef, message.Infof, oci.WithArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) From d6649a4e2fa01a293e31e24bcc957785e596872c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 23 Jan 2024 15:15:22 +0000 Subject: [PATCH 002/113] WIP --- src/pkg/oci/common.go | 1 - src/pkg/oci/copier.go | 7 +- src/pkg/oci/pull.go | 85 ------------------------ src/pkg/ocizarf/common.go | 36 ++++++++++ src/pkg/ocizarf/copier.go | 21 ++++++ src/pkg/ocizarf/fetch.go | 16 +++++ src/pkg/{zarfoci => ocizarf}/manifest.go | 2 +- src/pkg/{zarfoci => ocizarf}/pull.go | 4 +- src/pkg/packager/sources/oci.go | 4 +- 9 files changed, 85 insertions(+), 91 deletions(-) create mode 100644 src/pkg/ocizarf/common.go create mode 100644 src/pkg/ocizarf/copier.go create mode 100644 src/pkg/ocizarf/fetch.go rename src/pkg/{zarfoci => ocizarf}/manifest.go (97%) rename src/pkg/{zarfoci => ocizarf}/pull.go (98%) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index fe17993be8..dab3b2375f 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -94,7 +94,6 @@ func WithTargetPlatform(platform *ocispec.Platform) Modifier { } } -// WithSkeletonArch sets the target architecture for the remote to skeleton func WithSkeletonArch() Modifier { return WithTargetPlatform(&ocispec.Platform{ OS: MultiOS, diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index f74d0cd5ab..e34f51ba96 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -6,6 +6,7 @@ package oci import ( "context" + "encoding/json" "fmt" "io" "time" @@ -50,7 +51,11 @@ func CopyPackage(ctx context.Context, src *OrasRemote, dst *OrasRemote, start := time.Now() for idx, layer := range layers { - src.log("Copying layer:", message.JSONValue(layer)) + bytes, err := json.MarshalIndent(layer, "", " ") + if err != nil { + src.log("ERROR marshalling json: %s", err.Error()) + } + src.log("Copying layer:", string(bytes)) if err := sem.Acquire(ctx, 1); err != nil { return err } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 572a4d26ee..97d43f99b7 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -14,10 +14,8 @@ import ( "slices" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" @@ -218,86 +216,3 @@ func (o *OrasRemote) PullPackageMetadata(destinationDir string) ([]ocispec.Descr func (o *OrasRemote) PullPackageSBOM(destinationDir string) ([]ocispec.Descriptor, error) { return o.PullPackagePaths([]string{layout.SBOMTar}, destinationDir) } - -// LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. -// -// It also retrieves the descriptors for all image layers that are required by the components. -// -// It also respects the `required` flag on components, and will retrieve all necessary layers for required components. -func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) (layers []ocispec.Descriptor, err error) { - root, err := o.FetchRoot() - if err != nil { - return nil, err - } - - pkg, err := o.FetchZarfYAML() - if err != nil { - return nil, err - } - images := map[string]bool{} - tarballFormat := "%s.tar" - for _, name := range requestedComponents { - component := helpers.Find(pkg.Components, func(component types.ZarfComponent) bool { - return component.Name == name - }) - if component.Name == "" { - return nil, fmt.Errorf("component %s does not exist in this package", name) - } - } - for _, component := range pkg.Components { - // If we requested this component, or it is required, we need to pull its images and tarball - if slices.Contains(requestedComponents, component.Name) || component.Required { - for _, image := range component.Images { - images[image] = true - } - layers = append(layers, root.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf(tarballFormat, component.Name)))) - } - } - // Append the sboms.tar layer if it exists - // - // Since sboms.tar is not a heavy addition 99% of the time, we'll just always pull it - sbomsDescriptor := root.Locate(layout.SBOMTar) - if !IsEmptyDescriptor(sbomsDescriptor) { - layers = append(layers, sbomsDescriptor) - } - if len(images) > 0 { - // Add the image index and the oci-layout layers - layers = append(layers, root.Locate(ZarfPackageIndexPath), root.Locate(ZarfPackageLayoutPath)) - index, err := o.FetchImagesIndex() - if err != nil { - return nil, err - } - for image := range images { - // use docker's transform lib to parse the image ref - // this properly mirrors the logic within create - refInfo, err := transform.ParseImageRef(image) - if err != nil { - return nil, fmt.Errorf("failed to parse image ref %q: %w", image, err) - } - - manifestDescriptor := helpers.Find(index.Manifests, func(layer ocispec.Descriptor) bool { - return layer.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Reference || - // A backwards compatibility shim for older Zarf versions that would leave docker.io off of image annotations - (layer.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Path+refInfo.TagOrDigest && refInfo.Host == "docker.io") - }) - - // even though these are technically image manifests, we store them as Zarf blobs - manifestDescriptor.MediaType = ZarfLayerMediaTypeBlob - - manifest, err := o.FetchManifest(manifestDescriptor) - if err != nil { - return nil, err - } - // Add the manifest and the manifest config layers - layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifestDescriptor.Digest.Encoded()))) - layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifest.Config.Digest.Encoded()))) - - // Add all the layers from the manifest - for _, layer := range manifest.Layers { - layerPath := filepath.Join(ZarfPackageImagesBlobsDir, layer.Digest.Encoded()) - layers = append(layers, root.Locate(layerPath)) - } - } - } - return layers, nil -} diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go new file mode 100644 index 0000000000..2f1aafc09f --- /dev/null +++ b/src/pkg/ocizarf/common.go @@ -0,0 +1,36 @@ +package ocizarf + +import ( + "github.com/defenseunicorns/zarf/src/pkg/oci" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + // ZarfLayerMediaTypeBlob is the media type for all Zarf layers due to the range of possible content + ZarfLayerMediaTypeBlob = "application/vnd.zarf.layer.v1.blob" + // ZarfConfigMediaType is the media type for the manifest config + ZarfConfigMediaType = "application/vnd.zarf.config.v1+json" + // SkeletonArch is the architecture used for skeleton packages + SkeletonArch = "skeleton" + // MultiOS is the OS used for multi-platform packages + MultiOS = "multi" +) + +type ZarfOrasRemote struct { + oci.OrasRemote +} + +// log is a function that logs a message. +// type log func(string, ...any) + +// func NewZarfOrasRemote(url string, logger log, mod ...oci.Modifier) (*ZarfOrasRemote, error) { +// NewOrasRemote(url, logger, mod...) +// } + +// WithSkeletonArch sets the target architecture for the remote to skeleton +func WithSkeletonArch() oci.Modifier { + return oci.WithTargetPlatform(&ocispec.Platform{ + OS: MultiOS, + Architecture: SkeletonArch, + }) +} diff --git a/src/pkg/ocizarf/copier.go b/src/pkg/ocizarf/copier.go new file mode 100644 index 0000000000..14cae3f2d6 --- /dev/null +++ b/src/pkg/ocizarf/copier.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package ocizarf + +import ( + "context" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/oci" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func CopyPackage(ctx context.Context, src *oci.OrasRemote, dst *oci.OrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { + + if err := oci.CopyPackage(ctx, src, dst, nil, config.CommonOptions.OCIConcurrency); err != nil { + return err + } + return nil +} diff --git a/src/pkg/ocizarf/fetch.go b/src/pkg/ocizarf/fetch.go new file mode 100644 index 0000000000..15b0dcce60 --- /dev/null +++ b/src/pkg/ocizarf/fetch.go @@ -0,0 +1,16 @@ +package ocizarf + +import ( + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/types" +) + +// FetchZarfYAML fetches the zarf.yaml file from the remote repository. +func (o *ZarfOrasRemote) FetchZarfYAML() (pkg types.ZarfPackage, err error) { + manifest, err := o.FetchRoot() + if err != nil { + return pkg, err + } + return oci.FetchYAMLFile[types.ZarfPackage](o.FetchLayer, manifest, layout.ZarfYAML) +} diff --git a/src/pkg/zarfoci/manifest.go b/src/pkg/ocizarf/manifest.go similarity index 97% rename from src/pkg/zarfoci/manifest.go rename to src/pkg/ocizarf/manifest.go index 53a8dc3a9a..c82012a2fc 100644 --- a/src/pkg/zarfoci/manifest.go +++ b/src/pkg/ocizarf/manifest.go @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: 2021-Present The Zarf Authors // Package oci contains functions for interacting with Zarf packages stored in OCI registries. -package oci +package ocizarf import "path/filepath" diff --git a/src/pkg/zarfoci/pull.go b/src/pkg/ocizarf/pull.go similarity index 98% rename from src/pkg/zarfoci/pull.go rename to src/pkg/ocizarf/pull.go index 7ec4b89149..cd9ecf1d9e 100644 --- a/src/pkg/zarfoci/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: 2021-Present The Zarf Authors // Package oci contains functions for interacting with Zarf packages stored in OCI registries. -package oci +package ocizarf import ( "fmt" @@ -79,7 +79,7 @@ func LayersFromRequestedComponents(o *oci.OrasRemote, requestedComponents []stri }) // even though these are technically image manifests, we store them as Zarf blobs - manifestDescriptor.MediaType = oci.ZarfLayerMediaTypeBlob + manifestDescriptor.MediaType = ZarfLayerMediaTypeBlob manifest, err := o.FetchManifest(manifestDescriptor) if err != nil { diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 371c53a7b5..09bd88cf7a 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -44,7 +45,8 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er // pull only needed layers if --confirm is set if config.CommonOptions.Confirm { - layersToPull, err = s.LayersFromRequestedComponents(optionalComponents) + + layersToPull, err = ocizarf.LayersFromRequestedComponents(s.OrasRemote, optionalComponents) if err != nil { return fmt.Errorf("unable to get published component image layers: %s", err.Error()) } From 4dfa44c081e8df490ecd8261d4aea9a76729e615 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 23 Jan 2024 18:15:34 +0000 Subject: [PATCH 003/113] WIP, tests passing --- src/cmd/initialize.go | 3 +- src/cmd/tools/zarf.go | 3 +- src/pkg/oci/common.go | 1 + src/pkg/oci/pull.go | 39 +----------------------- src/pkg/ocizarf/common.go | 2 +- src/pkg/ocizarf/pull.go | 46 +++++++++++++++++++++++++++++ src/pkg/packager/sources/new.go | 3 +- src/pkg/packager/sources/oci.go | 4 +-- src/pkg/packager/sources/tarball.go | 4 +-- 9 files changed, 59 insertions(+), 46 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index c1dcd00681..4d96139e26 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -142,7 +143,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { if err != nil { return "", err } - source := sources.OCISource{OrasRemote: remote} + source := &sources.OCISource{ZarfOrasRemote: &ocizarf.ZarfOrasRemote{OrasRemote: remote}} return source.Collect(cacheDirectory) } // Otherwise, exit and tell the user to manually download the init-package diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index 6da3c64fa7..9d238d33a2 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -19,6 +19,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/pki" "github.com/defenseunicorns/zarf/src/types" @@ -188,7 +189,7 @@ var downloadInitCmd = &cobra.Command{ message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } - source := &sources.OCISource{OrasRemote: remote} + source := &sources.OCISource{ZarfOrasRemote: &ocizarf.ZarfOrasRemote{OrasRemote: remote}} _, err = source.Collect(outputDirectory) if err != nil { diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index dab3b2375f..c50507199e 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -36,6 +36,7 @@ const ( type log func(string, ...any) // OrasRemote is a wrapper around the Oras remote repository that includes a progress bar for interactive feedback. +// Do we want to start exporting fields in this struct? For example log may come in handy? type OrasRemote struct { repo *remote.Repository root *OCIManifest diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 97d43f99b7..017c654bee 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -13,7 +13,6 @@ import ( "slices" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -31,11 +30,6 @@ var ( ZarfPackageImagesBlobsDir = filepath.Join("images", "blobs", "sha256") ) -var ( - // PackageAlwaysPull is a list of paths that will always be pulled from the remote repository. - PackageAlwaysPull = []string{layout.ZarfYAML, layout.Checksums, layout.Signature} -) - // FileDescriptorExists returns true if the given file exists in the given directory with the expected SHA. func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDir string) bool { rel := desc.Annotations[ocispec.AnnotationTitle] @@ -69,29 +63,8 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi // // layersToPull is an optional parameter that allows the caller to specify which layers to pull. // -// The following layers will ALWAYS be pulled if they exist: -// - zarf.yaml -// - checksums.txt -// - zarf.yaml.sig +// ?! Now that we are going to 100% going to call this function with parameters do we still want layerstopull to be optional? func (o *OrasRemote) PullPackage(destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { - isPartialPull := len(layersToPull) > 0 - o.log("Pulling", o.repo.Reference) - - manifest, err := o.FetchRoot() - if err != nil { - return nil, err - } - - if isPartialPull { - for _, path := range PackageAlwaysPull { - desc := manifest.Locate(path) - layersToPull = append(layersToPull, desc) - } - } else { - layersToPull = append(layersToPull, manifest.Layers...) - } - layersToPull = append(layersToPull, manifest.Config) - // de-duplicate layers layersToPull = RemoveDuplicateDescriptors(layersToPull) @@ -206,13 +179,3 @@ func (o *OrasRemote) PullPackagePaths(paths []string, destinationDir string) ([] } return layersPulled, nil } - -// PullPackageMetadata pulls the package metadata from the remote repository and saves it to `destinationDir`. -func (o *OrasRemote) PullPackageMetadata(destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullPackagePaths(PackageAlwaysPull, destinationDir) -} - -// PullPackageSBOM pulls the package's sboms.tar from the remote repository and saves it to `destinationDir`. -func (o *OrasRemote) PullPackageSBOM(destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullPackagePaths([]string{layout.SBOMTar}, destinationDir) -} diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index 2f1aafc09f..919048c6b0 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -17,7 +17,7 @@ const ( ) type ZarfOrasRemote struct { - oci.OrasRemote + *oci.OrasRemote } // log is a function that logs a message. diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index cd9ecf1d9e..7b89883902 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -10,6 +10,7 @@ import ( "slices" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -17,6 +18,41 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) +var ( + // PackageAlwaysPull is a list of paths that will always be pulled from the remote repository. + PackageAlwaysPull = []string{layout.ZarfYAML, layout.Checksums, layout.Signature} +) + +// PullPackage pulls the package from the remote repository and saves it to the given path. +// +// layersToPull is an optional parameter that allows the caller to specify which layers to pull. +// +// The following layers will ALWAYS be pulled if they exist: +// - zarf.yaml +// - checksums.txt +// - zarf.yaml.sig +func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { + isPartialPull := len(layersToPull) > 0 + message.Infof("Pulling", o.Repo()) + + manifest, err := o.FetchRoot() + if err != nil { + return nil, err + } + + if isPartialPull { + for _, path := range PackageAlwaysPull { + desc := manifest.Locate(path) + layersToPull = append(layersToPull, desc) + } + } else { + layersToPull = append(layersToPull, manifest.Layers...) + } + layersToPull = append(layersToPull, manifest.Config) + + return o.OrasRemote.PullPackage(destinationDir, concurrency, layersToPull...) +} + // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. // It also retrieves the descriptors for all image layers that are required by the components. // @@ -98,3 +134,13 @@ func LayersFromRequestedComponents(o *oci.OrasRemote, requestedComponents []stri } return layers, nil } + +// PullPackageMetadata pulls the package metadata from the remote repository and saves it to `destinationDir`. +func (o *ZarfOrasRemote) PullPackageMetadata(destinationDir string) ([]ocispec.Descriptor, error) { + return o.PullPackagePaths(PackageAlwaysPull, destinationDir) +} + +// PullPackageSBOM pulls the package's sboms.tar from the remote repository and saves it to `destinationDir`. +func (o *ZarfOrasRemote) PullPackageSBOM(destinationDir string) ([]ocispec.Descriptor, error) { + return o.PullPackagePaths([]string{layout.SBOMTar}, destinationDir) +} diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index a9c7256c56..7debd8ebbb 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -13,6 +13,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -71,7 +72,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { if err != nil { return nil, err } - source = &OCISource{pkgOpts, remote} + source = &OCISource{pkgOpts, &ocizarf.ZarfOrasRemote{remote}} case "tarball": source = &TarballSource{pkgOpts} case "http", "https", "sget": diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 09bd88cf7a..a6f4b16543 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -31,7 +31,7 @@ var ( // OCISource is a package source for OCI registries. type OCISource struct { *types.ZarfPackageOptions - *oci.OrasRemote + *ocizarf.ZarfOrasRemote } // LoadPackage loads a package from an OCI registry. @@ -118,7 +118,7 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er func (s *OCISource) LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, skipValidation bool) (err error) { var pkg types.ZarfPackage - toPull := oci.PackageAlwaysPull + toPull := ocizarf.PackageAlwaysPull if wantSBOM { toPull = append(toPull, layout.SBOMTar) } diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index e65d0b962a..00006a1a9b 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" @@ -142,7 +142,7 @@ func (s *TarballSource) LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM b } } - toExtract := oci.PackageAlwaysPull + toExtract := ocizarf.PackageAlwaysPull if wantSBOM { toExtract = append(toExtract, layout.SBOMTar) } From c904cb0e7284d580ad0ae770a9377ecdc3b683a7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 23 Jan 2024 22:19:45 +0000 Subject: [PATCH 004/113] WIP, tests passing --- src/cmd/initialize.go | 4 +-- src/cmd/tools/zarf.go | 4 +-- src/pkg/oci/common.go | 7 ------ src/pkg/oci/copier.go | 4 +-- src/pkg/ociz/copier.go | 21 ++++++++++++++++ src/pkg/ocizarf/common.go | 29 ++++++++-------------- src/pkg/ocizarf/copier.go | 2 +- src/pkg/ocizarf/pull.go | 2 +- src/pkg/packager/composer/list.go | 4 +-- src/pkg/packager/composer/oci.go | 5 ++-- src/pkg/packager/create_stages.go | 5 ++-- src/pkg/packager/publish.go | 7 +++--- src/pkg/packager/sources/new.go | 4 +-- src/test/e2e/50_oci_publish_deploy_test.go | 8 +++--- 14 files changed, 58 insertions(+), 48 deletions(-) create mode 100644 src/pkg/ociz/copier.go diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 4d96139e26..7d0cd7aa68 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -139,11 +139,11 @@ func downloadInitPackage(cacheDirectory string) (string, error) { // If the user wants to download the init-package, download it if confirmDownload { - remote, err := oci.NewOrasRemote(url, message.Infof, oci.WithArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(url, oci.WithArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return "", err } - source := &sources.OCISource{ZarfOrasRemote: &ocizarf.ZarfOrasRemote{OrasRemote: remote}} + source := &sources.OCISource{ZarfOrasRemote: remote} return source.Collect(cacheDirectory) } // Otherwise, exit and tell the user to manually download the init-package diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index 9d238d33a2..e4442b1d50 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -184,12 +184,12 @@ var downloadInitCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { url := oci.GetInitPackageURL(config.CLIVersion) - remote, err := oci.NewOrasRemote(url, message.Infof, oci.WithArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(url, oci.WithArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } - source := &sources.OCISource{ZarfOrasRemote: &ocizarf.ZarfOrasRemote{OrasRemote: remote}} + source := &sources.OCISource{ZarfOrasRemote: remote} _, err = source.Collect(outputDirectory) if err != nil { diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index c50507199e..bbfc03effd 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -95,13 +95,6 @@ func WithTargetPlatform(platform *ocispec.Platform) Modifier { } } -func WithSkeletonArch() Modifier { - return WithTargetPlatform(&ocispec.Platform{ - OS: MultiOS, - Architecture: SkeletonArch, - }) -} - // WithArch sets the target architecture for the remote func WithArch(arch string) Modifier { return WithTargetPlatform(&ocispec.Platform{ diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index e34f51ba96..c487f06e8e 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -17,8 +17,8 @@ import ( "golang.org/x/sync/semaphore" ) -// CopyPackage copies a package from one OCI registry to another -func CopyPackage(ctx context.Context, src *OrasRemote, dst *OrasRemote, +// Copy copies an artifact from one OCI registry to another +func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { // create a new semaphore to limit concurrency sem := semaphore.NewWeighted(int64(concurrency)) diff --git a/src/pkg/ociz/copier.go b/src/pkg/ociz/copier.go new file mode 100644 index 0000000000..9ddb96df71 --- /dev/null +++ b/src/pkg/ociz/copier.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package zarfoci + +import ( + "context" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/oci" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func CopyPackage(ctx context.Context, src *oci.OrasRemote, dst *oci.OrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { + + if err := oci.Copy(ctx, src, dst, nil, config.CommonOptions.OCIConcurrency); err != nil { + return err + } + return nil +} diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index 919048c6b0..490aeb9513 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -1,36 +1,29 @@ package ocizarf import ( + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -const ( - // ZarfLayerMediaTypeBlob is the media type for all Zarf layers due to the range of possible content - ZarfLayerMediaTypeBlob = "application/vnd.zarf.layer.v1.blob" - // ZarfConfigMediaType is the media type for the manifest config - ZarfConfigMediaType = "application/vnd.zarf.config.v1+json" - // SkeletonArch is the architecture used for skeleton packages - SkeletonArch = "skeleton" - // MultiOS is the OS used for multi-platform packages - MultiOS = "multi" -) - type ZarfOrasRemote struct { *oci.OrasRemote } -// log is a function that logs a message. -// type log func(string, ...any) +type Modifier func(*oci.OrasRemote) -// func NewZarfOrasRemote(url string, logger log, mod ...oci.Modifier) (*ZarfOrasRemote, error) { -// NewOrasRemote(url, logger, mod...) -// } +func NewZarfOrasRemote(url string, mod ...oci.Modifier) (*ZarfOrasRemote, error) { + remote, err := oci.NewOrasRemote(url, message.Infof, mod...) + if err != nil { + return nil, err + } + return &ZarfOrasRemote{remote}, nil +} // WithSkeletonArch sets the target architecture for the remote to skeleton func WithSkeletonArch() oci.Modifier { return oci.WithTargetPlatform(&ocispec.Platform{ - OS: MultiOS, - Architecture: SkeletonArch, + OS: oci.MultiOS, + Architecture: oci.SkeletonArch, }) } diff --git a/src/pkg/ocizarf/copier.go b/src/pkg/ocizarf/copier.go index 14cae3f2d6..4748e105e5 100644 --- a/src/pkg/ocizarf/copier.go +++ b/src/pkg/ocizarf/copier.go @@ -14,7 +14,7 @@ import ( func CopyPackage(ctx context.Context, src *oci.OrasRemote, dst *oci.OrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { - if err := oci.CopyPackage(ctx, src, dst, nil, config.CommonOptions.OCIConcurrency); err != nil { + if err := oci.Copy(ctx, src, dst, nil, config.CommonOptions.OCIConcurrency); err != nil { return err } return nil diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index 7b89883902..c73c1e3f0d 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -115,7 +115,7 @@ func LayersFromRequestedComponents(o *oci.OrasRemote, requestedComponents []stri }) // even though these are technically image manifests, we store them as Zarf blobs - manifestDescriptor.MediaType = ZarfLayerMediaTypeBlob + manifestDescriptor.MediaType = oci.ZarfLayerMediaTypeBlob manifest, err := o.FetchManifest(manifestDescriptor) if err != nil { diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index f46235f212..caa26f821f 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -11,7 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -80,7 +80,7 @@ type ImportChain struct { head *Node tail *Node - remote *oci.OrasRemote + remote *ocizarf.ZarfOrasRemote } // Head returns the first node in the import chain diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index ba57996727..660f54e4f9 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -22,12 +23,12 @@ import ( ocistore "oras.land/oras-go/v2/content/oci" ) -func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { +func (ic *ImportChain) getRemote(url string) (*ocizarf.ZarfOrasRemote, error) { if ic.remote != nil { return ic.remote, nil } var err error - ic.remote, err = oci.NewOrasRemote(url, message.Infof, oci.WithSkeletonArch(), oci.WithInsecure(config.CommonOptions.Insecure)) + ic.remote, err = ocizarf.NewZarfOrasRemote(url, ocizarf.WithSkeletonArch(), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return nil, err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 5b0de8bf59..5095615e99 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -24,6 +24,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -250,7 +251,7 @@ func (p *Packager) output() error { if err != nil { return err } - remote, err := oci.NewOrasRemote(ref, message.Infof, oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(ref, oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return err } @@ -652,7 +653,7 @@ func (p *Packager) loadDifferentialData() error { // Load the package spec of the package we're using as a 'reference' for the differential build if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := oci.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, message.Infof, oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 2b359d881d..767c4117fe 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -37,7 +38,7 @@ func (p *Packager) Publish() (err error) { p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName arch := config.GetArch() - dstRemote, err := oci.NewOrasRemote(p.cfg.PublishOpts.PackageDestination, message.Infof, oci.WithArch(arch), oci.WithInsecure(config.CommonOptions.Insecure)) + dstRemote, err := ocizarf.NewZarfOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.WithArch(arch), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return err } @@ -57,7 +58,7 @@ func (p *Packager) Publish() (err error) { return fmt.Errorf("architecture mismatch (specified: %q, found %q)", arch, pkg.Build.Architecture) } - if err := oci.CopyPackage(ctx, srcRemote, dstRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { + if err := oci.Copy(ctx, srcRemote, dstRemote.OrasRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { return err } @@ -112,7 +113,7 @@ func (p *Packager) Publish() (err error) { return err } - remote, err := oci.NewOrasRemote(ref, message.Infof, oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(ref, oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return err } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 7debd8ebbb..eebb249857 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -68,11 +68,11 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } arch := config.GetArch() - remote, err := oci.NewOrasRemote(pkgSrc, message.Infof, oci.WithArch(arch), oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(pkgSrc, oci.WithArch(arch), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return nil, err } - source = &OCISource{pkgOpts, &ocizarf.ZarfOrasRemote{remote}} + source = &OCISource{pkgOpts, remote} case "tarball": source = &TarballSource{pkgOpts} case "http", "https", "sget": diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index d32950fbfd..d6dbafec47 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "oras.land/oras-go/v2/registry" @@ -129,10 +129,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - src, err := oci.NewOrasRemote(ref, message.Infof, oci.WithArch(e2e.Arch), oci.WithInsecure(true)) + src, err := ocizarf.NewZarfOrasRemote(ref, oci.WithArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) - dst, err := oci.NewOrasRemote(dstRef, message.Infof, oci.WithArch(e2e.Arch), oci.WithInsecure(true)) + dst, err := ocizarf.NewZarfOrasRemote(dstRef, oci.WithArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) @@ -150,7 +150,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { } require.Less(t, attempt, 5, "failed to ping registry") - err = oci.CopyPackage(ctx, src, dst, nil, 5) + err = oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, nil, 5) suite.NoError(err) srcRoot, err := src.FetchRoot() From 38308c270f8870344ced7a04cfe5bf2e769c4075 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 24 Jan 2024 13:55:58 +0000 Subject: [PATCH 005/113] moving concurrency tools to helpers --- src/internal/packager/images/pull.go | 7 ++++--- src/pkg/oci/common.go | 5 +++-- src/pkg/utils/{ => helpers}/concurrency.go | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) rename src/pkg/utils/{ => helpers}/concurrency.go (95%) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 0a7a0f4967..4bb2a8f0c4 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -19,6 +19,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" @@ -67,7 +68,7 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { logs.Warn.SetOutput(&message.DebugWriter{}) logs.Progress.SetOutput(&message.DebugWriter{}) - metadataImageConcurrency := utils.NewConcurrencyTools[ImgInfo, error](len(i.ImageList)) + metadataImageConcurrency := helpers.NewConcurrencyTools[ImgInfo, error](len(i.ImageList)) defer metadataImageConcurrency.Cancel() @@ -191,7 +192,7 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { // Spawn a goroutine for each layer to write it to disk using crane - layerWritingConcurrency := utils.NewConcurrencyTools[bool, error](len(processedLayers)) + layerWritingConcurrency := helpers.NewConcurrencyTools[bool, error](len(processedLayers)) defer layerWritingConcurrency.Cancel() @@ -335,7 +336,7 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { return nil, err } - imageSavingConcurrency := utils.NewConcurrencyTools[digestInfo, error](len(refInfoToImage)) + imageSavingConcurrency := helpers.NewConcurrencyTools[digestInfo, error](len(refInfoToImage)) defer imageSavingConcurrency.Cancel() diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 05beeb39d0..e948cd3ab4 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -113,8 +113,9 @@ func WithUserAgent(userAgent string) Modifier { // NewOrasRemote returns an oras remote repository client and context for the given url. // -// Registry auth is handled by the Docker CLI's credential store and checked before returning the client -// Should I keep platform in this function or only have it in NewZarfOrasRemote? +// # Registry auth is handled by the Docker CLI's credential store and checked before returning the client +// +// ?! Should I keep platform in this function or only have it in NewZarfOrasRemote? func NewOrasRemote(url string, logger log, platform ocispec.Platform, mods ...Modifier) (*OrasRemote, error) { ref, err := registry.ParseReference(strings.TrimPrefix(url, helpers.OCIURLPrefix)) if err != nil { diff --git a/src/pkg/utils/concurrency.go b/src/pkg/utils/helpers/concurrency.go similarity index 95% rename from src/pkg/utils/concurrency.go rename to src/pkg/utils/helpers/concurrency.go index 135d5caacc..9058e4adc4 100644 --- a/src/pkg/utils/concurrency.go +++ b/src/pkg/utils/helpers/concurrency.go @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: 2021-Present The Zarf Authors // forked from https://www.socketloop.com/tutorials/golang-byte-format-example -// Package utils provides generic helper functions. -package utils +// Package helpers provides generic helper functions with no external imports +package helpers import ( "context" From d13e39606266347c1ade8883482705b64523f0d6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 24 Jan 2024 15:39:10 +0000 Subject: [PATCH 006/113] inital progress bar abstraction --- src/pkg/message/progress.go | 23 +++++++++++++++++++++ src/pkg/oci/copier.go | 24 ++++++++++++++-------- src/pkg/ocizarf/copier.go | 7 ++++--- src/pkg/packager/publish.go | 2 +- src/test/e2e/50_oci_publish_deploy_test.go | 4 +++- 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/src/pkg/message/progress.go b/src/pkg/message/progress.go index a6c819093f..17b32362bc 100644 --- a/src/pkg/message/progress.go +++ b/src/pkg/message/progress.go @@ -37,6 +37,21 @@ func NewProgressBar(total int64, text string) *ProgressBar { } } +func (p *ProgressBar) Start(total int64, text string) { + p.startText = text + if NoProgress { + Info(p.startText) + } else { + p.progress, _ = pterm.DefaultProgressbar. + WithTotal(int(total)). + WithShowCount(false). + WithTitle(padding + text). + WithRemoveWhenDone(true). + WithMaxWidth(TermWidth). + Start() + } +} + // Update updates the ProgressBar with completed progress and new text. func (p *ProgressBar) Update(complete int64, text string) { if NoProgress { @@ -48,6 +63,10 @@ func (p *ProgressBar) Update(complete int64, text string) { p.Add(chunk) } +func (p *ProgressBar) Current() int64 { + return int64(p.progress.Current) +} + // UpdateTitle updates the ProgressBar with new text. func (p *ProgressBar) UpdateTitle(text string) { if NoProgress { @@ -78,6 +97,10 @@ func (p *ProgressBar) Write(data []byte) (int, error) { return n, nil } +func (p *ProgressBar) Finish(format string, a ...any) { + p.Successf(format, a...) +} + // Successf marks the ProgressBar as successful in the CLI. func (p *ProgressBar) Successf(format string, a ...any) { p.Stop() diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index c487f06e8e..89ae30f6ca 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -11,15 +11,22 @@ import ( "io" "time" - "github.com/defenseunicorns/zarf/src/pkg/message" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) +type ProgressTracker interface { + Start(total int64, text string) + Update(complete int64, text string) + Current() int64 + Write([]byte) (int, error) + Finish(format string, a ...any) +} + // Copy copies an artifact from one OCI registry to another func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, - include func(d ocispec.Descriptor) bool, concurrency int) error { + include func(d ocispec.Descriptor) bool, concurrency int, progressTracker ProgressTracker) error { // create a new semaphore to limit concurrency sem := semaphore.NewWeighted(int64(concurrency)) @@ -46,8 +53,10 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, } title := fmt.Sprintf("[0/%d] layers copied", len(layers)) - progressBar := message.NewProgressBar(size, title) - defer progressBar.Successf("Copied %s", src.repo.Reference) + + progressTracker.Start(size, title) + defer progressTracker.Finish("Copied %s", src.repo.Reference) + start := time.Now() for idx, layer := range layers { @@ -67,8 +76,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, } if exists { src.log("Layer already exists in destination, skipping") - progressBar.UpdateTitle(fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) - progressBar.Add(int(layer.Size)) + progressTracker.Update(layer.Size, fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) sem.Release(1) continue } @@ -103,7 +111,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, // this goroutine is responsible for updating the progressbar eg.Go(func() error { // read from the PipeReader to the progressbar - if _, err := io.Copy(progressBar, pr); err != nil { + if _, err := io.Copy(progressTracker, pr); err != nil { return fmt.Errorf("failed to update progress on layer %s: %w", layer.Digest, err) } return nil @@ -114,7 +122,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return err } sem.Release(1) - progressBar.UpdateTitle(fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) + progressTracker.Update(progressTracker.Current(), fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) } duration := time.Since(start) diff --git a/src/pkg/ocizarf/copier.go b/src/pkg/ocizarf/copier.go index 4748e105e5..a95723692b 100644 --- a/src/pkg/ocizarf/copier.go +++ b/src/pkg/ocizarf/copier.go @@ -7,14 +7,15 @@ package ocizarf import ( "context" - "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -func CopyPackage(ctx context.Context, src *oci.OrasRemote, dst *oci.OrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { +func CopyPackage(ctx context.Context, src *oci.OrasRemote, dst *oci.OrasRemote, + include func(d ocispec.Descriptor) bool, concurrency int) error { - if err := oci.Copy(ctx, src, dst, nil, config.CommonOptions.OCIConcurrency); err != nil { + if err := oci.Copy(ctx, src, dst, include, concurrency, &message.ProgressBar{}); err != nil { return err } return nil diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index df82873a33..fa5aeaee6f 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -58,7 +58,7 @@ func (p *Packager) Publish() (err error) { return fmt.Errorf("architecture mismatch (specified: %q, found %q)", arch, pkg.Build.Architecture) } - if err := oci.Copy(ctx, srcRemote, dstRemote.OrasRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { + if err := oci.Copy(ctx, srcRemote, dstRemote.OrasRemote, nil, config.CommonOptions.OCIConcurrency, &message.ProgressBar{}); err != nil { return err } diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index cce81ce2ec..caee57ace9 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/stretchr/testify/require" @@ -133,6 +134,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { suite.NoError(err) dst, err := ocizarf.NewZarfOrasRemote(dstRef, oci.PlatformForArch(e2e.Arch), oci.WithInsecure(true)) + suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) suite.NoError(err) @@ -149,7 +151,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { } require.Less(t, attempt, 5, "failed to ping registry") - err = oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, nil, 5) + err = oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, nil, 5, &message.ProgressBar{}) suite.NoError(err) srcRoot, err := src.FetchRoot() From 4b18cb61f4888dc3dada90a18d33b211c661a770 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 24 Jan 2024 15:49:33 +0000 Subject: [PATCH 007/113] WIP, tests passing --- src/pkg/packager/publish.go | 2 +- src/test/e2e/50_oci_publish_deploy_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index fa5aeaee6f..eebbc984ac 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -58,7 +58,7 @@ func (p *Packager) Publish() (err error) { return fmt.Errorf("architecture mismatch (specified: %q, found %q)", arch, pkg.Build.Architecture) } - if err := oci.Copy(ctx, srcRemote, dstRemote.OrasRemote, nil, config.CommonOptions.OCIConcurrency, &message.ProgressBar{}); err != nil { + if err := ocizarf.CopyPackage(ctx, srcRemote, dstRemote.OrasRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { return err } diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index caee57ace9..b472155010 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/stretchr/testify/require" @@ -151,7 +150,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { } require.Less(t, attempt, 5, "failed to ping registry") - err = oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, nil, 5, &message.ProgressBar{}) + err = ocizarf.CopyPackage(ctx, src.OrasRemote, dst.OrasRemote, nil, 5) suite.NoError(err) srcRoot, err := src.FetchRoot() From 236ce237ff0e64314c367bf7c8771db87f7f1baf Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 24 Jan 2024 16:12:50 +0000 Subject: [PATCH 008/113] WIP, move message to helpers --- src/pkg/message/message.go | 9 --------- src/pkg/message/progress.go | 8 ++++++++ src/pkg/oci/progress.go | 4 ++-- src/pkg/oci/push.go | 3 ++- src/pkg/utils/helpers/string.go | 14 ++++++++++++++ 5 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 src/pkg/utils/helpers/string.go diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index b4b6834a0f..d17baca298 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -357,15 +357,6 @@ func ColorWrap(str string, attr color.Attribute) string { return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) } -// First30last30 returns the source string that has been trimmed to 30 characters at the beginning and end. -func First30last30(s string) string { - if len(s) > 60 { - return s[0:27] + "..." + s[len(s)-26:] - } - - return s -} - func debugPrinter(offset int, a ...any) { printer := pterm.Debug.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) now := time.Now().Format(time.RFC3339) diff --git a/src/pkg/message/progress.go b/src/pkg/message/progress.go index 17b32362bc..d62483c3a8 100644 --- a/src/pkg/message/progress.go +++ b/src/pkg/message/progress.go @@ -97,8 +97,16 @@ func (p *ProgressBar) Write(data []byte) (int, error) { return n, nil } +// ?! Should we accept any? Should we accept any without calling the function Finishf? func (p *ProgressBar) Finish(format string, a ...any) { p.Successf(format, a...) + + // Maybe I will do something like this later + // if err != nil { + // spinner.Errorf(err, "Problem rendering the helm template for %s: %s", h.chart.Name, err.Error()) + // } else { + // spinner.Stop() + // } } // Successf marks the ProgressBar as successful in the CLI. diff --git a/src/pkg/oci/progress.go b/src/pkg/oci/progress.go index 7c262000f7..faf2cfa176 100644 --- a/src/pkg/oci/progress.go +++ b/src/pkg/oci/progress.go @@ -8,7 +8,7 @@ import ( "context" "fmt" - "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -27,7 +27,7 @@ func (o *OrasRemote) printLayer(desc ocispec.Descriptor, suffix string) error { title := desc.Annotations[ocispec.AnnotationTitle] var layerInfo string if title != "" { - layerInfo = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], message.First30last30(title)) + layerInfo = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], helpers.First30last30(title)) } else { layerInfo = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) } diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 958419c8a2..404ca9a6f4 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -119,7 +120,7 @@ func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.Packag // Get all of the layers in the package var descs []ocispec.Descriptor for name, path := range paths.Files() { - spinner.Updatef("Preparing layer %s", message.First30last30(name)) + spinner.Updatef("Preparing layer %s", helpers.First30last30(name)) mediaType := ZarfLayerMediaTypeBlob diff --git a/src/pkg/utils/helpers/string.go b/src/pkg/utils/helpers/string.go new file mode 100644 index 0000000000..ab6c69913f --- /dev/null +++ b/src/pkg/utils/helpers/string.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package helpers provides generic helper functions with no external imports +package helpers + +// First30last30 returns the source string that has been trimmed to 30 characters at the beginning and end. +func First30last30(s string) string { + if len(s) > 60 { + return s[0:27] + "..." + s[len(s)-26:] + } + + return s +} From 5592aaf47458b0b638f7ad9cbbab91cbff070a1c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 24 Jan 2024 17:00:51 +0000 Subject: [PATCH 009/113] Allowing nil progress bar --- src/pkg/oci/copier.go | 62 +++++++++++++++++++++++++++-------------- src/pkg/ocizarf/pull.go | 2 +- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 89ae30f6ca..46de3f3273 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -22,9 +22,11 @@ type ProgressTracker interface { Current() int64 Write([]byte) (int, error) Finish(format string, a ...any) + Add(n int) } // Copy copies an artifact from one OCI registry to another +// ?! We should probably add in nil checks for progressTracker I assume func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, include func(d ocispec.Descriptor) bool, concurrency int, progressTracker ProgressTracker) error { // create a new semaphore to limit concurrency @@ -52,10 +54,11 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, size += layer.Size } - title := fmt.Sprintf("[0/%d] layers copied", len(layers)) - - progressTracker.Start(size, title) - defer progressTracker.Finish("Copied %s", src.repo.Reference) + if progressTracker != nil { + title := fmt.Sprintf("[0/%d] layers copied", len(layers)) + progressTracker.Start(size, title) + defer progressTracker.Finish("Copied %s", src.repo.Reference) + } start := time.Now() @@ -76,28 +79,52 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, } if exists { src.log("Layer already exists in destination, skipping") - progressTracker.Update(layer.Size, fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) + if progressTracker != nil { + progressTracker.Update(layer.Size, fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) + } sem.Release(1) continue } - // create a new pipe so we can write to both the progressbar and the destination at the same time - pr, pw := io.Pipe() - eg, ectx := errgroup.WithContext(ctx) eg.SetLimit(2) + var tr io.Reader + var pw *io.PipeWriter + // fetch the layer from the source rc, err := src.repo.Fetch(ectx, layer) if err != nil { return err } - // TeeReader gets the data from the fetching layer and writes it to the PipeWriter - tr := io.TeeReader(rc, pw) + + // ?! Does changing the order of this affect anything + if progressTracker != nil { + // create a new pipe so we can write to both the progressbar and the destination at the same time + pr, pwTemp := io.Pipe() + pw = pwTemp + + // TeeReader gets the data from the fetching layer and writes it to the PipeWriter + tr = io.TeeReader(rc, pw) + // this goroutine is responsible for updating the progressbar + eg.Go(func() error { + // read from the PipeReader to the progressbar + if _, err := io.Copy(progressTracker, pr); err != nil { + return fmt.Errorf("failed to update progress on layer %s: %w", layer.Digest, err) + } + return nil + }) + } else { + tr = rc + } // this goroutine is responsible for pushing the layer to the destination eg.Go(func() error { - defer pw.Close() + defer func() { + if pw != nil { + pw.Close() + } + }() // get data from the TeeReader and push it to the destination // push the layer to the destination @@ -108,21 +135,14 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return nil }) - // this goroutine is responsible for updating the progressbar - eg.Go(func() error { - // read from the PipeReader to the progressbar - if _, err := io.Copy(progressTracker, pr); err != nil { - return fmt.Errorf("failed to update progress on layer %s: %w", layer.Digest, err) - } - return nil - }) - // wait for the goroutines to finish if err := eg.Wait(); err != nil { return err } sem.Release(1) - progressTracker.Update(progressTracker.Current(), fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) + if progressTracker != nil { + progressTracker.Update(progressTracker.Current(), fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) + } } duration := time.Since(start) diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index c73c1e3f0d..0e52cb9c23 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -33,7 +33,7 @@ var ( // - zarf.yaml.sig func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { isPartialPull := len(layersToPull) > 0 - message.Infof("Pulling", o.Repo()) + message.Debugf("Pulling %s", o.Repo().Reference) manifest, err := o.FetchRoot() if err != nil { From 9b529dbc26a184b693973f46c2283d1d9f3772d9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 24 Jan 2024 17:19:27 +0000 Subject: [PATCH 010/113] Cleaner implemention of allowing nil --- src/pkg/oci/copier.go | 46 ++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 46de3f3273..159862434a 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -89,42 +89,21 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, eg, ectx := errgroup.WithContext(ctx) eg.SetLimit(2) - var tr io.Reader - var pw *io.PipeWriter - // fetch the layer from the source rc, err := src.repo.Fetch(ectx, layer) if err != nil { return err } - // ?! Does changing the order of this affect anything - if progressTracker != nil { - // create a new pipe so we can write to both the progressbar and the destination at the same time - pr, pwTemp := io.Pipe() - pw = pwTemp - - // TeeReader gets the data from the fetching layer and writes it to the PipeWriter - tr = io.TeeReader(rc, pw) - // this goroutine is responsible for updating the progressbar - eg.Go(func() error { - // read from the PipeReader to the progressbar - if _, err := io.Copy(progressTracker, pr); err != nil { - return fmt.Errorf("failed to update progress on layer %s: %w", layer.Digest, err) - } - return nil - }) - } else { - tr = rc - } + // create a new pipe so we can write to both the progressbar and the destination at the same time + pr, pw := io.Pipe() + + // TeeReader gets the data from the fetching layer and writes it to the PipeWriter + tr := io.TeeReader(rc, pw) // this goroutine is responsible for pushing the layer to the destination eg.Go(func() error { - defer func() { - if pw != nil { - pw.Close() - } - }() + defer pw.Close() // get data from the TeeReader and push it to the destination // push the layer to the destination @@ -135,6 +114,19 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return nil }) + // this goroutine is responsible for updating the progressbar + eg.Go(func() error { + writer := io.Discard + if progressTracker != nil { + writer = progressTracker + } + // read from the PipeReader to the progressbar + if _, err := io.Copy(writer, pr); err != nil { + return fmt.Errorf("failed to update progress on layer %s: %w", layer.Digest, err) + } + return nil + }) + // wait for the goroutines to finish if err := eg.Wait(); err != nil { return err From 7e845e574b51d4a17244eb6033e8abf9e80da32c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 24 Jan 2024 21:20:23 +0000 Subject: [PATCH 011/113] WIP, moving to prgoressbar interface --- src/pkg/message/progress.go | 16 ++++++---------- src/pkg/oci/copier.go | 8 ++++---- src/pkg/oci/push.go | 8 +++----- src/pkg/packager/create_stages.go | 2 +- src/pkg/packager/publish.go | 2 +- src/pkg/utils/transport.go | 10 ++++++---- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/pkg/message/progress.go b/src/pkg/message/progress.go index d62483c3a8..a38a71c7b3 100644 --- a/src/pkg/message/progress.go +++ b/src/pkg/message/progress.go @@ -97,16 +97,12 @@ func (p *ProgressBar) Write(data []byte) (int, error) { return n, nil } -// ?! Should we accept any? Should we accept any without calling the function Finishf? -func (p *ProgressBar) Finish(format string, a ...any) { - p.Successf(format, a...) - - // Maybe I will do something like this later - // if err != nil { - // spinner.Errorf(err, "Problem rendering the helm template for %s: %s", h.chart.Name, err.Error()) - // } else { - // spinner.Stop() - // } +func (p *ProgressBar) Finish(err error, format string, a ...any) { + if err != nil { + p.Errorf(err, format, a...) + } else { + p.Successf(format, a...) + } } // Successf marks the ProgressBar as successful in the CLI. diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 159862434a..335ca0c640 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -16,19 +16,19 @@ import ( "golang.org/x/sync/semaphore" ) -type ProgressTracker interface { +type ProgressBar interface { Start(total int64, text string) Update(complete int64, text string) Current() int64 Write([]byte) (int, error) - Finish(format string, a ...any) + Finish(err error, format string, a ...any) Add(n int) } // Copy copies an artifact from one OCI registry to another // ?! We should probably add in nil checks for progressTracker I assume func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, - include func(d ocispec.Descriptor) bool, concurrency int, progressTracker ProgressTracker) error { + include func(d ocispec.Descriptor) bool, concurrency int, progressTracker ProgressBar) (err error) { // create a new semaphore to limit concurrency sem := semaphore.NewWeighted(int64(concurrency)) @@ -57,7 +57,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, if progressTracker != nil { title := fmt.Sprintf("[0/%d] layers copied", len(layers)) progressTracker.Start(size, title) - defer progressTracker.Finish("Copied %s", src.repo.Reference) + defer progressTracker.Finish(err, "Copied %s", src.repo.Reference) } start := time.Now() diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 404ca9a6f4..bf8f05777c 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -8,7 +8,6 @@ import ( "bytes" "encoding/json" "errors" - "fmt" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -104,7 +103,7 @@ func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descr } // PublishPackage publishes the package to the remote repository. -func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int) error { +func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int, progressBar ProgressBar) (err error) { ctx := o.ctx // source file store src, err := file.New(paths.Base) @@ -155,8 +154,8 @@ func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.Packag } total += root.Size + manifestConfigDesc.Size - o.Transport.ProgressBar = message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.repo.Reference.Repository, o.repo.Reference.Reference)) - defer o.Transport.ProgressBar.Stop() + o.Transport.ProgressBar = progressBar + defer o.Transport.ProgressBar.Finish(err, "Published %s [%s]", o.repo.Reference, root.MediaType) publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), o.repo, "", copyOpts) if err != nil { @@ -166,7 +165,6 @@ func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.Packag if err := o.UpdateIndex(o.repo.Reference.Reference, pkg.Build.Architecture, publishedDesc); err != nil { return err } - o.Transport.ProgressBar.Successf("Published %s [%s]", o.repo.Reference, root.MediaType) return nil } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index bf52353d92..6d1f9cab6b 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -256,7 +256,7 @@ func (p *Packager) output() error { return err } - err = remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) + err = remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency, &message.ProgressBar{}) if err != nil { return fmt.Errorf("unable to publish package: %w", err) } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index eebbc984ac..4a8a12e965 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -128,7 +128,7 @@ func (p *Packager) Publish() (err error) { message.HeaderInfof("📦 PACKAGE PUBLISH %s:%s", p.cfg.Pkg.Metadata.Name, ref) // Publish the package/skeleton to the registry - if err := remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency); err != nil { + if err := remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency, &message.ProgressBar{}); err != nil { return err } if p.cfg.CreateOpts.IsSkeleton { diff --git a/src/pkg/utils/transport.go b/src/pkg/utils/transport.go index 3d31d819d6..c88ded79e5 100644 --- a/src/pkg/utils/transport.go +++ b/src/pkg/utils/transport.go @@ -13,10 +13,12 @@ import ( ) type progressBar interface { - Add(int) - Write([]byte) (n int, err error) - Stop() - Successf(format string, args ...interface{}) + Start(total int64, text string) + Update(complete int64, text string) + Current() int64 + Write([]byte) (int, error) + Finish(err error, format string, a ...any) + Add(n int) } // Transport is an http.RoundTripper that keeps track of the in-flight From 02a650de87a21703fe6fbe42818f2bbe8d3c6a4d Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 25 Jan 2024 14:50:58 +0000 Subject: [PATCH 012/113] refactoring message bar in oci --- src/pkg/message/progress.go | 15 ----- src/pkg/oci/copier.go | 64 ++++++++-------------- src/pkg/oci/manifest.go | 13 +++++ src/pkg/oci/push.go | 5 +- src/pkg/ocizarf/copier.go | 14 ++++- src/pkg/utils/transport.go | 16 ++---- src/test/e2e/50_oci_publish_deploy_test.go | 2 +- 7 files changed, 59 insertions(+), 70 deletions(-) diff --git a/src/pkg/message/progress.go b/src/pkg/message/progress.go index a38a71c7b3..c9df5a1e24 100644 --- a/src/pkg/message/progress.go +++ b/src/pkg/message/progress.go @@ -37,21 +37,6 @@ func NewProgressBar(total int64, text string) *ProgressBar { } } -func (p *ProgressBar) Start(total int64, text string) { - p.startText = text - if NoProgress { - Info(p.startText) - } else { - p.progress, _ = pterm.DefaultProgressbar. - WithTotal(int(total)). - WithShowCount(false). - WithTitle(padding + text). - WithRemoveWhenDone(true). - WithMaxWidth(TermWidth). - Start() - } -} - // Update updates the ProgressBar with completed progress and new text. func (p *ProgressBar) Update(complete int64, text string) { if NoProgress { diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 335ca0c640..9090dd0f2b 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -6,6 +6,7 @@ package oci import ( "context" + "encoding/binary" "encoding/json" "fmt" "io" @@ -16,19 +17,25 @@ import ( "golang.org/x/sync/semaphore" ) -type ProgressBar interface { - Start(total int64, text string) - Update(complete int64, text string) - Current() int64 - Write([]byte) (int, error) - Finish(err error, format string, a ...any) - Add(n int) +func (DiscardProgressWriter) Write(p []byte) (int, error) { + return len(p), nil +} + +func (DiscardProgressWriter) UpdateTitle(s string) {} + +type DiscardProgressWriter struct{} + +type ProgressWriter interface { + UpdateTitle(string) + io.Writer } // Copy copies an artifact from one OCI registry to another -// ?! We should probably add in nil checks for progressTracker I assume func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, - include func(d ocispec.Descriptor) bool, concurrency int, progressTracker ProgressBar) (err error) { + include func(d ocispec.Descriptor) bool, concurrency int, progressBar ProgressWriter) (err error) { + if progressBar == nil { + progressBar = DiscardProgressWriter{} + } // create a new semaphore to limit concurrency sem := semaphore.NewWeighted(int64(concurrency)) @@ -38,27 +45,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return err } - var layers []ocispec.Descriptor - for _, layer := range srcRoot.Layers { - if include != nil && include(layer) { - layers = append(layers, layer) - } else if include == nil { - layers = append(layers, layer) - } - } - - layers = append(layers, srcRoot.Config) - - size := int64(0) - for _, layer := range layers { - size += layer.Size - } - - if progressTracker != nil { - title := fmt.Sprintf("[0/%d] layers copied", len(layers)) - progressTracker.Start(size, title) - defer progressTracker.Finish(err, "Copied %s", src.repo.Reference) - } + layers := srcRoot.GetLayers(include) start := time.Now() @@ -79,9 +66,10 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, } if exists { src.log("Layer already exists in destination, skipping") - if progressTracker != nil { - progressTracker.Update(layer.Size, fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) - } + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, uint64(b[layer.Size])) + progressBar.Write(b) + progressBar.UpdateTitle(fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) sem.Release(1) continue } @@ -116,12 +104,8 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, // this goroutine is responsible for updating the progressbar eg.Go(func() error { - writer := io.Discard - if progressTracker != nil { - writer = progressTracker - } // read from the PipeReader to the progressbar - if _, err := io.Copy(writer, pr); err != nil { + if _, err := io.Copy(progressBar, pr); err != nil { return fmt.Errorf("failed to update progress on layer %s: %w", layer.Digest, err) } return nil @@ -132,9 +116,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return err } sem.Release(1) - if progressTracker != nil { - progressTracker.Update(progressTracker.Current(), fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) - } + progressBar.UpdateTitle(fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) } duration := time.Since(start) diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index ca8c559443..1c2e77ba39 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -46,3 +46,16 @@ func (m *OCIManifest) SumLayersSize() int64 { func (m *OCIManifest) MarshalJSON() ([]byte, error) { return json.Marshal(m.Manifest) } + +func (m *OCIManifest) GetLayers(include func(d ocispec.Descriptor) bool) []ocispec.Descriptor { + var layers []ocispec.Descriptor + for _, layer := range m.Layers { + if include != nil && include(layer) { + layers = append(layers, layer) + } else if include == nil { + layers = append(layers, layer) + } + } + layers = append(layers, m.Config) + return layers +} diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index bf8f05777c..a9396e01df 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -103,7 +103,8 @@ func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descr } // PublishPackage publishes the package to the remote repository. -func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int, progressBar ProgressBar) (err error) { +// TODO it's a go semantic to have ctx as the first arguement of all these functions +func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int, progressBar ProgressWriter) (err error) { ctx := o.ctx // source file store src, err := file.New(paths.Base) @@ -155,7 +156,7 @@ func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.Packag total += root.Size + manifestConfigDesc.Size o.Transport.ProgressBar = progressBar - defer o.Transport.ProgressBar.Finish(err, "Published %s [%s]", o.repo.Reference, root.MediaType) + //defer o.Transport.ProgressBar.Finish(err, "Published %s [%s]", o.repo.Reference, root.MediaType) publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), o.repo, "", copyOpts) if err != nil { diff --git a/src/pkg/ocizarf/copier.go b/src/pkg/ocizarf/copier.go index a95723692b..8b581fa208 100644 --- a/src/pkg/ocizarf/copier.go +++ b/src/pkg/ocizarf/copier.go @@ -6,6 +6,7 @@ package ocizarf import ( "context" + "fmt" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" @@ -15,7 +16,18 @@ import ( func CopyPackage(ctx context.Context, src *oci.OrasRemote, dst *oci.OrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { - if err := oci.Copy(ctx, src, dst, include, concurrency, &message.ProgressBar{}); err != nil { + srcRoot, err := src.FetchRoot() + if err != nil { + return err + } + layers := srcRoot.GetLayers(include) + size := srcRoot.SumLayersSize() + + title := fmt.Sprintf("[0/%d] layers copied", len(layers)) + progressBar := message.NewProgressBar(size, title) + defer progressBar.Finish(err, "Copied %s", src.Repo().Reference) + + if err = oci.Copy(ctx, src, dst, include, concurrency, progressBar); err != nil { return err } return nil diff --git a/src/pkg/utils/transport.go b/src/pkg/utils/transport.go index c88ded79e5..acaf23ced4 100644 --- a/src/pkg/utils/transport.go +++ b/src/pkg/utils/transport.go @@ -12,24 +12,20 @@ import ( "oras.land/oras-go/v2/registry/remote/retry" ) -type progressBar interface { - Start(total int64, text string) - Update(complete int64, text string) - Current() int64 - Write([]byte) (int, error) - Finish(err error, format string, a ...any) - Add(n int) +type progressWriter interface { + UpdateTitle(string) + io.Writer } // Transport is an http.RoundTripper that keeps track of the in-flight // request and add hooks to report upload progress. type Transport struct { Base http.RoundTripper - ProgressBar progressBar + ProgressBar progressWriter } // NewTransport returns a custom transport that tracks an http.RoundTripper and a message.ProgressBar. -func NewTransport(base http.RoundTripper, bar progressBar) *Transport { +func NewTransport(base http.RoundTripper, bar progressWriter) *Transport { return &Transport{ Base: base, ProgressBar: bar, @@ -98,7 +94,7 @@ func (t *Transport) roundTrip(req *http.Request) (resp *http.Response, err error if resp != nil && req.Method == http.MethodHead && err == nil && t.ProgressBar != nil { if resp.ContentLength > 0 { - t.ProgressBar.Add(int(resp.ContentLength)) + // t.ProgressBar.Add(int(resp.ContentLength)) } } return resp, err diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index b472155010..baf2f57f29 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -56,7 +56,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_0_Publish() { ref := suite.Reference.String() stdOut, stdErr, err := e2e.Zarf("package", "publish", example, "oci://"+ref, "--insecure") suite.NoError(err, stdOut, stdErr) - suite.Contains(stdErr, "Published "+ref) + //suite.Contains(stdErr, "Published "+ref) // Pull the package via OCI. stdOut, stdErr, err = e2e.Zarf("package", "pull", "oci://"+ref+"/helm-charts:0.0.1", "--insecure") From ec0bc527717fe6a020f7f27273a1828d9a29ef74 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 29 Jan 2024 16:46:29 +0000 Subject: [PATCH 013/113] WIP, testing works, progress bar numbers not right --- src/pkg/oci/common.go | 22 ++++++- src/pkg/oci/copier.go | 13 ---- src/pkg/oci/push.go | 49 ++++----------- src/pkg/ocizarf/push.go | 73 ++++++++++++++++++++++ src/pkg/packager/create_stages.go | 4 +- src/pkg/packager/publish.go | 3 +- src/test/e2e/50_oci_publish_deploy_test.go | 2 +- 7 files changed, 111 insertions(+), 55 deletions(-) create mode 100644 src/pkg/ocizarf/push.go diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index e948cd3ab4..3bdee9328e 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -7,6 +7,7 @@ package oci import ( "context" "fmt" + "io" "net/http" "strings" @@ -32,6 +33,25 @@ const ( MultiOS = "multi" ) +func (DiscardProgressWriter) Write(p []byte) (int, error) { + return len(p), nil +} + +func (DiscardProgressWriter) UpdateTitle(s string) {} + +type DiscardProgressWriter struct{} + +type ProgressWriter interface { + UpdateTitle(string) + io.Writer +} + +type metadata struct { + title string + description string + architecture string +} + // log is a function that logs a message. type log func(string, ...any) @@ -104,7 +124,7 @@ func PlatformForArch(arch string) ocispec.Platform { } } -// WithArch sets the target architecture for the remote +// WithUserAgent sets the target architecture for the remote func WithUserAgent(userAgent string) Modifier { return func(o *OrasRemote) { o.userAgent = userAgent diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 9090dd0f2b..456b4bd994 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -17,19 +17,6 @@ import ( "golang.org/x/sync/semaphore" ) -func (DiscardProgressWriter) Write(p []byte) (int, error) { - return len(p), nil -} - -func (DiscardProgressWriter) UpdateTitle(s string) {} - -type DiscardProgressWriter struct{} - -type ProgressWriter interface { - UpdateTitle(string) - io.Writer -} - // Copy copies an artifact from one OCI registry to another func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, include func(d ocispec.Descriptor) bool, concurrency int, progressBar ProgressWriter) (err error) { diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index a9396e01df..fb2e964ca7 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -6,12 +6,10 @@ package oci import ( "bytes" + "context" "encoding/json" "errors" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -48,6 +46,7 @@ func (o *OrasRemote) pushManifestConfigFromMetadata(metadata *types.ZarfMetadata ocispec.AnnotationDescription: metadata.Description, } manifestConfig := ConfigPartial{ + // ?! Is there any reason we use build arch over o.platform.arch? Architecture: build.Architecture, OCIVersion: "1.0.1", Annotations: annotations, @@ -104,40 +103,9 @@ func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descr // PublishPackage publishes the package to the remote repository. // TODO it's a go semantic to have ctx as the first arguement of all these functions -func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int, progressBar ProgressWriter) (err error) { - ctx := o.ctx - // source file store - src, err := file.New(paths.Base) - if err != nil { - return err - } - defer src.Close() - - message.Infof("Publishing package to %s", o.repo.Reference) - spinner := message.NewProgressSpinner("") - defer spinner.Stop() - - // Get all of the layers in the package - var descs []ocispec.Descriptor - for name, path := range paths.Files() { - spinner.Updatef("Preparing layer %s", helpers.First30last30(name)) - - mediaType := ZarfLayerMediaTypeBlob - - desc, err := src.Add(ctx, name, mediaType, path) - if err != nil { - return err - } - descs = append(descs, desc) - } - spinner.Successf("Prepared all layers") - +func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, pkg *types.ZarfPackage, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - var total int64 - for _, desc := range descs { - total += desc.Size - } // assumes referrers API is not supported since OCI artifact // media type is not supported o.repo.SetReferrersCapability(false) @@ -145,18 +113,23 @@ func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.Packag // push the manifest config // since this config is so tiny, and the content is not used again // it is not logged to the progress, but will error if it fails + + // ?! Would it make sense to have a metadata OCI object? manifestConfigDesc, err := o.pushManifestConfigFromMetadata(&pkg.Metadata, &pkg.Build) if err != nil { return err } - root, err := o.generatePackManifest(src, descs, &manifestConfigDesc, &pkg.Metadata) + root, err := o.generatePackManifest(src, desc, &manifestConfigDesc, &pkg.Metadata) if err != nil { return err } - total += root.Size + manifestConfigDesc.Size + // ?! Why do we add the size of the root when we already have the descripters? + // total += root.Size + manifestConfigDesc.Size o.Transport.ProgressBar = progressBar - //defer o.Transport.ProgressBar.Finish(err, "Published %s [%s]", o.repo.Reference, root.MediaType) + + // ?! I considered adding a function like finish. I'm leaning towards leaving it out and leaving this up to the implementer + // defer o.Transport.ProgressBar.Finish(err, "Published %s [%s]", o.repo.Reference, root.MediaType) publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), o.repo, "", copyOpts) if err != nil { diff --git a/src/pkg/ocizarf/push.go b/src/pkg/ocizarf/push.go new file mode 100644 index 0000000000..39963989c9 --- /dev/null +++ b/src/pkg/ocizarf/push.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package ocizarf + +import ( + "context" + "fmt" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content/file" +) + +func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int) error { + src, err := file.New(paths.Base) + if err != nil { + return err + } + defer src.Close() + + message.Infof("Publishing package to %s", o.Repo().Reference) + spinner := message.NewProgressSpinner("") + defer spinner.Stop() + + // Get all of the layers in the package + var descs []ocispec.Descriptor + for name, path := range paths.Files() { + spinner.Updatef("Preparing layer %s", helpers.First30last30(name)) + + mediaType := oci.ZarfLayerMediaTypeBlob + + desc, err := src.Add(ctx, name, mediaType, path) + if err != nil { + return err + } + descs = append(descs, desc) + } + spinner.Successf("Prepared all layers") + + copyOpts := o.CopyOpts + copyOpts.Concurrency = concurrency + var total int64 + for _, desc := range descs { + total += desc.Size + } + // manifestConfigDesc, err := o.pushManifestConfigFromMetadata(&pkg.Metadata, &pkg.Build) + // if err != nil { + // return err + // } + // root, err := o.generatePackManifest(src, descs, &manifestConfigDesc, &pkg.Metadata) + // if err != nil { + // return err + // } + // total += root.Size + manifestConfigDesc.Size + + progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) + err = o.PublishPackage(ctx, src, pkg, descs, config.CommonOptions.OCIConcurrency, progressBar) + if err != nil { + progressBar.Stop() + return fmt.Errorf("unable to publish package: %w", err) + } + + // ?! Do I know the media type 100% at this point + progressBar.Successf("Published %s [%s]", o.Repo().Reference, oci.ZarfLayerMediaTypeBlob) + return nil +} diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 6d1f9cab6b..2e20fc6f8c 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -5,6 +5,7 @@ package packager import ( + "context" "errors" "fmt" "os" @@ -256,7 +257,8 @@ func (p *Packager) output() error { return err } - err = remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency, &message.ProgressBar{}) + ctx := context.TODO() + err = remote.PublishZarfPackage(ctx, &p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) if err != nil { return fmt.Errorf("unable to publish package: %w", err) } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 4a8a12e965..e1becd79d2 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -128,7 +128,8 @@ func (p *Packager) Publish() (err error) { message.HeaderInfof("📦 PACKAGE PUBLISH %s:%s", p.cfg.Pkg.Metadata.Name, ref) // Publish the package/skeleton to the registry - if err := remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency, &message.ProgressBar{}); err != nil { + ctx := context.TODO() + if err := remote.PublishZarfPackage(ctx, &p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency); err != nil { return err } if p.cfg.CreateOpts.IsSkeleton { diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index baf2f57f29..b472155010 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -56,7 +56,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_0_Publish() { ref := suite.Reference.String() stdOut, stdErr, err := e2e.Zarf("package", "publish", example, "oci://"+ref, "--insecure") suite.NoError(err, stdOut, stdErr) - //suite.Contains(stdErr, "Published "+ref) + suite.Contains(stdErr, "Published "+ref) // Pull the package via OCI. stdOut, stdErr, err = e2e.Zarf("package", "pull", "oci://"+ref+"/helm-charts:0.0.1", "--insecure") From 4c718eefd8d8c40fa4bc7303e5d3c0b82d74db56 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 29 Jan 2024 19:35:02 +0000 Subject: [PATCH 014/113] WIP, tests are working. Zarf out of push.go --- src/pkg/oci/common.go | 6 ---- src/pkg/oci/pull.go | 6 ++-- src/pkg/oci/push.go | 65 ++++++++++++----------------------------- src/pkg/ocizarf/pull.go | 2 +- src/pkg/ocizarf/push.go | 39 ++++++++++++++++++------- 5 files changed, 50 insertions(+), 68 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 3bdee9328e..6cd1617b6c 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -46,12 +46,6 @@ type ProgressWriter interface { io.Writer } -type metadata struct { - title string - description string - architecture string -} - // log is a function that logs a message. type log func(string, ...any) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 017c654bee..3e5ad59dab 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -59,12 +59,10 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi return actual == desc.Digest.Encoded() } -// PullPackage pulls the package from the remote repository and saves it to the given path. +// PullLayers pulls the package from the remote repository and saves it to the given path. // // layersToPull is an optional parameter that allows the caller to specify which layers to pull. -// -// ?! Now that we are going to 100% going to call this function with parameters do we still want layerstopull to be optional? -func (o *OrasRemote) PullPackage(destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { +func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, layersToPull []ocispec.Descriptor) ([]ocispec.Descriptor, error) { // de-duplicate layers layersToPull = RemoveDuplicateDescriptors(layersToPull) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index fb2e964ca7..7876e82ac2 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -10,7 +10,6 @@ import ( "encoding/json" "errors" - "github.com/defenseunicorns/zarf/src/types" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" @@ -40,14 +39,9 @@ func (o *OrasRemote) PushLayer(b []byte, mediaType string) (ocispec.Descriptor, } // pushManifestConfigFromMetadata pushes the manifest config with metadata to the remote repository. -func (o *OrasRemote) pushManifestConfigFromMetadata(metadata *types.ZarfMetadata, build *types.ZarfBuildData) (ocispec.Descriptor, error) { - annotations := map[string]string{ - ocispec.AnnotationTitle: metadata.Name, - ocispec.AnnotationDescription: metadata.Description, - } +func (o *OrasRemote) pushManifestConfigFromMetadata(annotations map[string]string) (ocispec.Descriptor, error) { manifestConfig := ConfigPartial{ - // ?! Is there any reason we use build arch over o.platform.arch? - Architecture: build.Architecture, + Architecture: o.targetPlatform.Architecture, OCIVersion: "1.0.1", Annotations: annotations, } @@ -55,39 +49,16 @@ func (o *OrasRemote) pushManifestConfigFromMetadata(metadata *types.ZarfMetadata if err != nil { return ocispec.Descriptor{}, err } - return o.PushLayer(manifestConfigBytes, ZarfConfigMediaType) + // If Media type is not set it will be set to the default + // ?! Is this the image config media type or the + return o.PushLayer(manifestConfigBytes, annotations[ocispec.MediaTypeImageConfig]) } -// manifestAnnotationsFromMetadata returns the annotations for the manifest from the given metadata. -func (o *OrasRemote) manifestAnnotationsFromMetadata(metadata *types.ZarfMetadata) map[string]string { - annotations := map[string]string{ - ocispec.AnnotationDescription: metadata.Description, - } - - if url := metadata.URL; url != "" { - annotations[ocispec.AnnotationURL] = url - } - if authors := metadata.Authors; authors != "" { - annotations[ocispec.AnnotationAuthors] = authors - } - if documentation := metadata.Documentation; documentation != "" { - annotations[ocispec.AnnotationDocumentation] = documentation - } - if source := metadata.Source; source != "" { - annotations[ocispec.AnnotationSource] = source - } - if vendor := metadata.Vendor; vendor != "" { - annotations[ocispec.AnnotationVendor] = vendor - } - - return annotations -} - -func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descriptor, configDesc *ocispec.Descriptor, metadata *types.ZarfMetadata) (ocispec.Descriptor, error) { +func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descriptor, configDesc *ocispec.Descriptor, annotations map[string]string) (ocispec.Descriptor, error) { packOpts := oras.PackManifestOptions{ Layers: descs, ConfigDescriptor: configDesc, - ManifestAnnotations: o.manifestAnnotationsFromMetadata(metadata), + ManifestAnnotations: annotations, } root, err := oras.PackManifest(o.ctx, src, oras.PackManifestVersion1_1_RC4, "", packOpts) @@ -102,8 +73,11 @@ func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descr } // PublishPackage publishes the package to the remote repository. -// TODO it's a go semantic to have ctx as the first arguement of all these functions -func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, pkg *types.ZarfPackage, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { +// TODO: We need documentation to Library that they need to use the ocispec package for map keys +func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annotations map[string]string, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { + if annotations[ocispec.AnnotationTitle] == "" { + return errors.New("invalid annotations: please include ocispec.AnnotationTitle") + } copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency // assumes referrers API is not supported since OCI artifact @@ -113,30 +87,27 @@ func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, pkg *t // push the manifest config // since this config is so tiny, and the content is not used again // it is not logged to the progress, but will error if it fails - - // ?! Would it make sense to have a metadata OCI object? - manifestConfigDesc, err := o.pushManifestConfigFromMetadata(&pkg.Metadata, &pkg.Build) + manifestConfigDesc, err := o.pushManifestConfigFromMetadata(annotations) if err != nil { return err } - root, err := o.generatePackManifest(src, desc, &manifestConfigDesc, &pkg.Metadata) + root, err := o.generatePackManifest(src, desc, &manifestConfigDesc, annotations) if err != nil { return err } - // ?! Why do we add the size of the root when we already have the descripters? + + // ?! Do we care about the size of these? + // Maybe we should add a total function or simliar to set the byte amount in the progress bar // total += root.Size + manifestConfigDesc.Size o.Transport.ProgressBar = progressBar - // ?! I considered adding a function like finish. I'm leaning towards leaving it out and leaving this up to the implementer - // defer o.Transport.ProgressBar.Finish(err, "Published %s [%s]", o.repo.Reference, root.MediaType) - publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), o.repo, "", copyOpts) if err != nil { return err } - if err := o.UpdateIndex(o.repo.Reference.Reference, pkg.Build.Architecture, publishedDesc); err != nil { + if err := o.UpdateIndex(o.repo.Reference.Reference, o.targetPlatform.Architecture, publishedDesc); err != nil { return err } diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index 0e52cb9c23..371cc6a6db 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -50,7 +50,7 @@ func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, lay } layersToPull = append(layersToPull, manifest.Config) - return o.OrasRemote.PullPackage(destinationDir, concurrency, layersToPull...) + return o.PullLayers(destinationDir, concurrency, layersToPull) } // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. diff --git a/src/pkg/ocizarf/push.go b/src/pkg/ocizarf/push.go index 39963989c9..379a22bcb6 100644 --- a/src/pkg/ocizarf/push.go +++ b/src/pkg/ocizarf/push.go @@ -50,18 +50,10 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf for _, desc := range descs { total += desc.Size } - // manifestConfigDesc, err := o.pushManifestConfigFromMetadata(&pkg.Metadata, &pkg.Build) - // if err != nil { - // return err - // } - // root, err := o.generatePackManifest(src, descs, &manifestConfigDesc, &pkg.Metadata) - // if err != nil { - // return err - // } - // total += root.Size + manifestConfigDesc.Size progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) - err = o.PublishPackage(ctx, src, pkg, descs, config.CommonOptions.OCIConcurrency, progressBar) + annotations := zarfPackageOciAnnotations(&pkg.Metadata) + err = o.PublishPackage(ctx, src, annotations, descs, config.CommonOptions.OCIConcurrency, progressBar) if err != nil { progressBar.Stop() return fmt.Errorf("unable to publish package: %w", err) @@ -71,3 +63,30 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf progressBar.Successf("Published %s [%s]", o.Repo().Reference, oci.ZarfLayerMediaTypeBlob) return nil } + +func zarfPackageOciAnnotations(metadata *types.ZarfMetadata) map[string]string { + + annotations := map[string]string{ + ocispec.AnnotationTitle: metadata.Name, + ocispec.AnnotationDescription: metadata.Description, + ocispec.MediaTypeImageConfig: oci.ZarfConfigMediaType, + } + + if url := metadata.URL; url != "" { + annotations[ocispec.AnnotationURL] = url + } + if authors := metadata.Authors; authors != "" { + annotations[ocispec.AnnotationAuthors] = authors + } + if documentation := metadata.Documentation; documentation != "" { + annotations[ocispec.AnnotationDocumentation] = documentation + } + if source := metadata.Source; source != "" { + annotations[ocispec.AnnotationSource] = source + } + if vendor := metadata.Vendor; vendor != "" { + annotations[ocispec.AnnotationVendor] = vendor + } + + return annotations +} From 7c9444ce16e9ea515d8917f8053c3f0c373b0f9f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 30 Jan 2024 14:30:25 +0000 Subject: [PATCH 015/113] arch fix --- src/pkg/oci/push.go | 8 +++++--- src/pkg/ocizarf/push.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 7876e82ac2..ffcf5b54b8 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -43,7 +43,8 @@ func (o *OrasRemote) pushManifestConfigFromMetadata(annotations map[string]strin manifestConfig := ConfigPartial{ Architecture: o.targetPlatform.Architecture, OCIVersion: "1.0.1", - Annotations: annotations, + // ?! Should this be all the users annotations or just title and description ? + Annotations: annotations, } manifestConfigBytes, err := json.Marshal(manifestConfig) if err != nil { @@ -74,7 +75,8 @@ func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descr // PublishPackage publishes the package to the remote repository. // TODO: We need documentation to Library that they need to use the ocispec package for map keys -func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annotations map[string]string, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { +// TODO: ?! We need the package.build architecutre rather than the o.targetPlatform.Arch. How do we make this clear to users? +func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annotations map[string]string, arch string, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { if annotations[ocispec.AnnotationTitle] == "" { return errors.New("invalid annotations: please include ocispec.AnnotationTitle") } @@ -107,7 +109,7 @@ func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annota return err } - if err := o.UpdateIndex(o.repo.Reference.Reference, o.targetPlatform.Architecture, publishedDesc); err != nil { + if err := o.UpdateIndex(o.repo.Reference.Reference, arch, publishedDesc); err != nil { return err } diff --git a/src/pkg/ocizarf/push.go b/src/pkg/ocizarf/push.go index 379a22bcb6..aadcaa5bf2 100644 --- a/src/pkg/ocizarf/push.go +++ b/src/pkg/ocizarf/push.go @@ -53,7 +53,7 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) annotations := zarfPackageOciAnnotations(&pkg.Metadata) - err = o.PublishPackage(ctx, src, annotations, descs, config.CommonOptions.OCIConcurrency, progressBar) + err = o.PublishPackage(ctx, src, annotations, pkg.Build.Architecture, descs, config.CommonOptions.OCIConcurrency, progressBar) if err != nil { progressBar.Stop() return fmt.Errorf("unable to publish package: %w", err) From d24d38a370988bbde21417099d22aacbcae3f9a6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 30 Jan 2024 14:32:14 +0000 Subject: [PATCH 016/113] WIP Refactoring zarf out, tests passing --- src/pkg/oci/fetch.go | 11 ----------- src/pkg/ocizarf/copier.go | 4 ++-- src/pkg/ocizarf/pull.go | 2 +- src/pkg/packager/publish.go | 4 ++-- src/pkg/packager/sources/oci.go | 2 +- src/test/e2e/50_oci_publish_deploy_test.go | 2 +- 6 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/pkg/oci/fetch.go b/src/pkg/oci/fetch.go index 7ca32cd0a3..4ae50ac7ab 100644 --- a/src/pkg/oci/fetch.go +++ b/src/pkg/oci/fetch.go @@ -8,8 +8,6 @@ import ( "encoding/json" "fmt" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" @@ -71,15 +69,6 @@ func (o *OrasRemote) FetchLayer(desc ocispec.Descriptor) (bytes []byte, err erro return content.FetchAll(o.ctx, o.repo, desc) } -// FetchZarfYAML fetches the zarf.yaml file from the remote repository. -func (o *OrasRemote) FetchZarfYAML() (pkg types.ZarfPackage, err error) { - manifest, err := o.FetchRoot() - if err != nil { - return pkg, err - } - return FetchYAMLFile[types.ZarfPackage](o.FetchLayer, manifest, layout.ZarfYAML) -} - // FetchImagesIndex fetches the images/index.json file from the remote repository. func (o *OrasRemote) FetchImagesIndex() (index *ocispec.Index, err error) { manifest, err := o.FetchRoot() diff --git a/src/pkg/ocizarf/copier.go b/src/pkg/ocizarf/copier.go index 8b581fa208..6a0e88bce1 100644 --- a/src/pkg/ocizarf/copier.go +++ b/src/pkg/ocizarf/copier.go @@ -13,7 +13,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -func CopyPackage(ctx context.Context, src *oci.OrasRemote, dst *oci.OrasRemote, +func CopyPackage(ctx context.Context, src *ZarfOrasRemote, dst *ZarfOrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { srcRoot, err := src.FetchRoot() @@ -27,7 +27,7 @@ func CopyPackage(ctx context.Context, src *oci.OrasRemote, dst *oci.OrasRemote, progressBar := message.NewProgressBar(size, title) defer progressBar.Finish(err, "Copied %s", src.Repo().Reference) - if err = oci.Copy(ctx, src, dst, include, concurrency, progressBar); err != nil { + if err = oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, include, concurrency, progressBar); err != nil { return err } return nil diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index 371cc6a6db..676e9bd2c3 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -57,7 +57,7 @@ func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, lay // It also retrieves the descriptors for all image layers that are required by the components. // // It also respects the `required` flag on components, and will retrieve all necessary layers for required components. -func LayersFromRequestedComponents(o *oci.OrasRemote, requestedComponents []string) (layers []ocispec.Descriptor, err error) { +func LayersFromRequestedComponents(o *ZarfOrasRemote, requestedComponents []string) (layers []ocispec.Descriptor, err error) { root, err := o.FetchRoot() if err != nil { return nil, err diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index e1becd79d2..c7cb6eedbe 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -30,7 +30,7 @@ func (p *Packager) Publish() (err error) { ctx := context.TODO() // oci --> oci is a special case, where we will use oci.CopyPackage so that we can transfer the package // w/o layers touching the filesystem - srcRemote := p.source.(*sources.OCISource).OrasRemote + srcRemote := p.source.(*sources.OCISource).ZarfOrasRemote parts := strings.Split(srcRemote.Repo().Reference.Repository, "/") packageName := parts[len(parts)-1] @@ -58,7 +58,7 @@ func (p *Packager) Publish() (err error) { return fmt.Errorf("architecture mismatch (specified: %q, found %q)", arch, pkg.Build.Architecture) } - if err := ocizarf.CopyPackage(ctx, srcRemote, dstRemote.OrasRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { + if err := ocizarf.CopyPackage(ctx, srcRemote, dstRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { return err } diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index a6f4b16543..dc21c6a270 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -46,7 +46,7 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er // pull only needed layers if --confirm is set if config.CommonOptions.Confirm { - layersToPull, err = ocizarf.LayersFromRequestedComponents(s.OrasRemote, optionalComponents) + layersToPull, err = ocizarf.LayersFromRequestedComponents(s.ZarfOrasRemote, optionalComponents) if err != nil { return fmt.Errorf("unable to get published component image layers: %s", err.Error()) } diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index b472155010..3dfa2d4115 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -150,7 +150,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { } require.Less(t, attempt, 5, "failed to ping registry") - err = ocizarf.CopyPackage(ctx, src.OrasRemote, dst.OrasRemote, nil, 5) + err = ocizarf.CopyPackage(ctx, src, dst, nil, 5) suite.NoError(err) srcRoot, err := src.FetchRoot() From d1bbde02046ee8353a81e9719355d6deb03cfe4a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 30 Jan 2024 16:31:07 +0000 Subject: [PATCH 017/113] WIP, tests passing --- src/pkg/oci/copier.go | 7 +++---- src/pkg/oci/interface.go | 11 ----------- src/pkg/ocizarf/push.go | 1 + src/pkg/utils/transport.go | 4 +++- 4 files changed, 7 insertions(+), 16 deletions(-) delete mode 100644 src/pkg/oci/interface.go diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 456b4bd994..5d139a31ca 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -6,7 +6,6 @@ package oci import ( "context" - "encoding/binary" "encoding/json" "fmt" "io" @@ -53,9 +52,9 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, } if exists { src.log("Layer already exists in destination, skipping") - b := make([]byte, 8) - binary.LittleEndian.PutUint64(b, uint64(b[layer.Size])) - progressBar.Write(b) + contentLength := layer.Size + data := make([]byte, contentLength) + progressBar.Write(data) progressBar.UpdateTitle(fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) sem.Release(1) continue diff --git a/src/pkg/oci/interface.go b/src/pkg/oci/interface.go deleted file mode 100644 index 4d67fcb102..0000000000 --- a/src/pkg/oci/interface.go +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. -package oci - -type OrasObj interface { - Pull() - Push() - New() -} diff --git a/src/pkg/ocizarf/push.go b/src/pkg/ocizarf/push.go index aadcaa5bf2..4e4467b3d9 100644 --- a/src/pkg/ocizarf/push.go +++ b/src/pkg/ocizarf/push.go @@ -51,6 +51,7 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf total += desc.Size } + message.Infof("total is %d", total) progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) annotations := zarfPackageOciAnnotations(&pkg.Metadata) err = o.PublishPackage(ctx, src, annotations, pkg.Build.Architecture, descs, config.CommonOptions.OCIConcurrency, progressBar) diff --git a/src/pkg/utils/transport.go b/src/pkg/utils/transport.go index acaf23ced4..85d5a5c65d 100644 --- a/src/pkg/utils/transport.go +++ b/src/pkg/utils/transport.go @@ -94,7 +94,9 @@ func (t *Transport) roundTrip(req *http.Request) (resp *http.Response, err error if resp != nil && req.Method == http.MethodHead && err == nil && t.ProgressBar != nil { if resp.ContentLength > 0 { - // t.ProgressBar.Add(int(resp.ContentLength)) + contentLength := int(resp.ContentLength) + data := make([]byte, contentLength) + t.ProgressBar.Write(data) } } return resp, err From 4a60079a3b9edf85d3f48a2b66edf19d9a050fc2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 30 Jan 2024 16:50:23 +0000 Subject: [PATCH 018/113] WIP, refactoring out zarf --- src/pkg/oci/fetch.go | 9 --------- src/pkg/oci/pull.go | 2 ++ src/pkg/ocizarf/fetch.go | 10 ++++++++++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/pkg/oci/fetch.go b/src/pkg/oci/fetch.go index 4ae50ac7ab..0e0333b1e9 100644 --- a/src/pkg/oci/fetch.go +++ b/src/pkg/oci/fetch.go @@ -69,15 +69,6 @@ func (o *OrasRemote) FetchLayer(desc ocispec.Descriptor) (bytes []byte, err erro return content.FetchAll(o.ctx, o.repo, desc) } -// FetchImagesIndex fetches the images/index.json file from the remote repository. -func (o *OrasRemote) FetchImagesIndex() (index *ocispec.Index, err error) { - manifest, err := o.FetchRoot() - if err != nil { - return index, err - } - return FetchJSONFile[*ocispec.Index](o.FetchLayer, manifest, ZarfPackageIndexPath) -} - // FetchJSONFile fetches the given JSON file from the remote repository. func FetchJSONFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *OCIManifest, path string) (result T, err error) { descriptor := manifest.Locate(path) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 3e5ad59dab..8d09040c2b 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -141,6 +141,8 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta // PullLayer pulls a layer from the remote repository and saves it to `destinationDir/annotationTitle`. func (o *OrasRemote) PullLayer(desc ocispec.Descriptor, destinationDir string) error { + // ?! Would this not fail in a different way if this error wasn't here + // Is this simply for better error messaging? if desc.MediaType != ZarfLayerMediaTypeBlob { return fmt.Errorf("invalid media type for file layer: %s", desc.MediaType) } diff --git a/src/pkg/ocizarf/fetch.go b/src/pkg/ocizarf/fetch.go index 15b0dcce60..65d58c0662 100644 --- a/src/pkg/ocizarf/fetch.go +++ b/src/pkg/ocizarf/fetch.go @@ -4,6 +4,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/types" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // FetchZarfYAML fetches the zarf.yaml file from the remote repository. @@ -14,3 +15,12 @@ func (o *ZarfOrasRemote) FetchZarfYAML() (pkg types.ZarfPackage, err error) { } return oci.FetchYAMLFile[types.ZarfPackage](o.FetchLayer, manifest, layout.ZarfYAML) } + +// FetchImagesIndex fetches the images/index.json file from the remote repository. +func (o *ZarfOrasRemote) FetchImagesIndex() (index *ocispec.Index, err error) { + manifest, err := o.FetchRoot() + if err != nil { + return index, err + } + return oci.FetchJSONFile[*ocispec.Index](o.FetchLayer, manifest, ZarfPackageIndexPath) +} From 06e52f91b03bb84296f2fd43ab3d20256cdcd1fa Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 30 Jan 2024 17:26:51 +0000 Subject: [PATCH 019/113] WIP, refactoring zarf out of oci --- src/cmd/initialize.go | 2 +- src/cmd/tools/zarf.go | 2 +- src/pkg/oci/pull.go | 9 ------- src/pkg/oci/utils.go | 36 ------------------------- src/pkg/ocizarf/manifest.go | 16 ----------- src/pkg/ocizarf/pull.go | 6 +++++ src/pkg/ocizarf/utils.go | 45 +++++++++++++++++++++++++++++++ src/pkg/packager/create_stages.go | 2 +- src/pkg/packager/publish.go | 2 +- 9 files changed, 55 insertions(+), 65 deletions(-) delete mode 100644 src/pkg/ocizarf/manifest.go create mode 100644 src/pkg/ocizarf/utils.go diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index d9fe1e054a..f37ca92c53 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -120,7 +120,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { } var confirmDownload bool - url := oci.GetInitPackageURL(config.CLIVersion) + url := ocizarf.GetInitPackageURL(config.CLIVersion) // Give the user the choice to download the init-package and note that this does require an internet connection message.Question(fmt.Sprintf(lang.CmdInitPullAsk, url)) diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index aa62baba49..d7910a1a25 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -182,7 +182,7 @@ var downloadInitCmd = &cobra.Command{ Use: "download-init", Short: lang.CmdToolsDownloadInitShort, Run: func(cmd *cobra.Command, args []string) { - url := oci.GetInitPackageURL(config.CLIVersion) + url := ocizarf.GetInitPackageURL(config.CLIVersion) remote, err := ocizarf.NewZarfOrasRemote(url, oci.PlatformForArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 8d09040c2b..7734efa6cc 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -21,15 +21,6 @@ import ( "oras.land/oras-go/v2/content/file" ) -var ( - // ZarfPackageIndexPath is the path to the index.json file in the OCI package. - ZarfPackageIndexPath = filepath.Join("images", "index.json") - // ZarfPackageLayoutPath is the path to the oci-layout file in the OCI package. - ZarfPackageLayoutPath = filepath.Join("images", "oci-layout") - // ZarfPackageImagesBlobsDir is the path to the directory containing the image blobs in the OCI package. - ZarfPackageImagesBlobsDir = filepath.Join("images", "blobs", "sha256") -) - // FileDescriptorExists returns true if the given file exists in the given directory with the expected SHA. func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDir string) bool { rel := desc.Annotations[ocispec.AnnotationTitle] diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index 73019ba46d..20e541ebf6 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -5,45 +5,14 @@ package oci import ( - "errors" "fmt" "strings" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/registry" ) -// ReferenceFromMetadata returns a reference for the given metadata. -func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata, build *types.ZarfBuildData) (string, error) { - ver := metadata.Version - if len(ver) == 0 { - return "", errors.New("version is required for publishing") - } - - if !strings.HasSuffix(registryLocation, "/") { - registryLocation = registryLocation + "/" - } - registryLocation = strings.TrimPrefix(registryLocation, helpers.OCIURLPrefix) - - format := "%s%s:%s" - raw := fmt.Sprintf(format, registryLocation, metadata.Name, ver) - - if build != nil && build.Flavor != "" { - raw = fmt.Sprintf("%s-%s", raw, build.Flavor) - } - - //o.log("Raw OCI reference from metadata:", raw) - - ref, err := registry.ParseReference(raw) - if err != nil { - return "", err - } - - return ref.String(), nil -} - // IsEmptyDescriptor returns true if the given descriptor is empty. func IsEmptyDescriptor(desc ocispec.Descriptor) bool { return desc.Digest == "" && desc.Size == 0 @@ -74,8 +43,3 @@ func RemoveDuplicateDescriptors(descriptors []ocispec.Descriptor) []ocispec.Desc } return list } - -// GetInitPackageURL returns the URL for the init package for the given version. -func GetInitPackageURL(version string) string { - return fmt.Sprintf("ghcr.io/defenseunicorns/packages/init:%s", version) -} diff --git a/src/pkg/ocizarf/manifest.go b/src/pkg/ocizarf/manifest.go deleted file mode 100644 index c82012a2fc..0000000000 --- a/src/pkg/ocizarf/manifest.go +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. -package ocizarf - -import "path/filepath" - -var ( - // ZarfPackageIndexPath is the path to the index.json file in the OCI package. - ZarfPackageIndexPath = filepath.Join("images", "index.json") - // ZarfPackageLayoutPath is the path to the oci-layout file in the OCI package. - ZarfPackageLayoutPath = filepath.Join("images", "oci-layout") - // ZarfPackageImagesBlobsDir is the path to the directory containing the image blobs in the OCI package. - ZarfPackageImagesBlobsDir = filepath.Join("images", "blobs", "sha256") -) diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index 676e9bd2c3..7b1e77ef1d 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -21,6 +21,12 @@ import ( var ( // PackageAlwaysPull is a list of paths that will always be pulled from the remote repository. PackageAlwaysPull = []string{layout.ZarfYAML, layout.Checksums, layout.Signature} + // ZarfPackageIndexPath is the path to the index.json file in the OCI package. + ZarfPackageIndexPath = filepath.Join("images", "index.json") + // ZarfPackageImagesBlobsDir is the path to the directory containing the image blobs in the OCI package. + ZarfPackageImagesBlobsDir = filepath.Join("images", "blobs", "sha256") + // ZarfPackageLayoutPath is the path to the oci-layout file in the OCI package. + ZarfPackageLayoutPath = filepath.Join("images", "oci-layout") ) // PullPackage pulls the package from the remote repository and saves it to the given path. diff --git a/src/pkg/ocizarf/utils.go b/src/pkg/ocizarf/utils.go new file mode 100644 index 0000000000..3652359ed5 --- /dev/null +++ b/src/pkg/ocizarf/utils.go @@ -0,0 +1,45 @@ +package ocizarf + +import ( + "errors" + "fmt" + "strings" + + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + "oras.land/oras-go/v2/registry" +) + +// ReferenceFromMetadata returns a reference for the given metadata. +func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata, build *types.ZarfBuildData) (string, error) { + ver := metadata.Version + if len(ver) == 0 { + return "", errors.New("version is required for publishing") + } + + if !strings.HasSuffix(registryLocation, "/") { + registryLocation = registryLocation + "/" + } + registryLocation = strings.TrimPrefix(registryLocation, helpers.OCIURLPrefix) + + format := "%s%s:%s" + raw := fmt.Sprintf(format, registryLocation, metadata.Name, ver) + + if build != nil && build.Flavor != "" { + raw = fmt.Sprintf("%s-%s", raw, build.Flavor) + } + + //o.log("Raw OCI reference from metadata:", raw) + + ref, err := registry.ParseReference(raw) + if err != nil { + return "", err + } + + return ref.String(), nil +} + +// GetInitPackageURL returns the URL for the init package for the given version. +func GetInitPackageURL(version string) string { + return fmt.Sprintf("ghcr.io/defenseunicorns/packages/init:%s", version) +} diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 2e20fc6f8c..16712d4292 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -248,7 +248,7 @@ func (p *Packager) output() error { // Create a remote ref + client for the package (if output is OCI) // then publish the package to the remote. if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { - ref, err := oci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) + ref, err := ocizarf.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index c7cb6eedbe..33aea59831 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -108,7 +108,7 @@ func (p *Packager) Publish() (err error) { } // Get a reference to the registry for this package - ref, err := oci.ReferenceFromMetadata(p.cfg.PublishOpts.PackageDestination, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) + ref, err := ocizarf.ReferenceFromMetadata(p.cfg.PublishOpts.PackageDestination, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) if err != nil { return err } From d85124960664a93da3a0937f7d6dfc648bcd5a6a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 30 Jan 2024 20:16:33 +0000 Subject: [PATCH 020/113] fixing platform --- src/pkg/oci/common.go | 12 ++++++++-- src/pkg/oci/push.go | 44 ++++++++++++++----------------------- src/pkg/ocizarf/common.go | 3 ++- src/pkg/ocizarf/push.go | 2 -- src/pkg/packager/publish.go | 11 +++++++--- 5 files changed, 37 insertions(+), 35 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 6cd1617b6c..6e83c0a90e 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -37,7 +37,7 @@ func (DiscardProgressWriter) Write(p []byte) (int, error) { return len(p), nil } -func (DiscardProgressWriter) UpdateTitle(s string) {} +func (DiscardProgressWriter) UpdateTitle(_ string) {} type DiscardProgressWriter struct{} @@ -60,6 +60,7 @@ type OrasRemote struct { targetPlatform *ocispec.Platform userAgent string log log + mediaType string } // Modifier is a function that modifies an OrasRemote @@ -102,7 +103,7 @@ func WithInsecureSkipVerify(insecure bool) Modifier { } } -// PlatformForSkeleton sets the target architecture for the remote to skeleton +// PlatformForSkeleton returns a skeleton oci func PlatformForSkeleton() ocispec.Platform { return ocispec.Platform{ OS: MultiOS, @@ -110,6 +111,13 @@ func PlatformForSkeleton() ocispec.Platform { } } +// WithMediaType sets the mediatype for the remote +func WithMediaType(mediaType string) Modifier { + return func(o *OrasRemote) { + o.mediaType = mediaType + } +} + // PlatformForArch sets the target architecture for the remote func PlatformForArch(arch string) ocispec.Platform { return ocispec.Platform{ diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index ffcf5b54b8..aa0ff09e15 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -33,26 +34,27 @@ type ConfigPartial struct { } // PushLayer pushes the given layer (bytes) to the remote repository. -func (o *OrasRemote) PushLayer(b []byte, mediaType string) (ocispec.Descriptor, error) { +func (o *OrasRemote) PushLayer(b []byte, mediaType string) (*ocispec.Descriptor, error) { desc := content.NewDescriptorFromBytes(mediaType, b) - return desc, o.repo.Push(o.ctx, desc, bytes.NewReader(b)) + return &desc, o.repo.Push(o.ctx, desc, bytes.NewReader(b)) } // pushManifestConfigFromMetadata pushes the manifest config with metadata to the remote repository. -func (o *OrasRemote) pushManifestConfigFromMetadata(annotations map[string]string) (ocispec.Descriptor, error) { +func (o *OrasRemote) pushManifestConfigFromMetadata(annotations map[string]string) (*ocispec.Descriptor, error) { + if annotations[ocispec.AnnotationTitle] == "" { + return nil, fmt.Errorf("invalid annotations: please include value for %q", ocispec.AnnotationTitle) + } manifestConfig := ConfigPartial{ Architecture: o.targetPlatform.Architecture, OCIVersion: "1.0.1", - // ?! Should this be all the users annotations or just title and description ? - Annotations: annotations, + Annotations: annotations, } manifestConfigBytes, err := json.Marshal(manifestConfig) if err != nil { - return ocispec.Descriptor{}, err + return nil, err } // If Media type is not set it will be set to the default - // ?! Is this the image config media type or the - return o.PushLayer(manifestConfigBytes, annotations[ocispec.MediaTypeImageConfig]) + return o.PushLayer(manifestConfigBytes, o.mediaType) } func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descriptor, configDesc *ocispec.Descriptor, annotations map[string]string) (ocispec.Descriptor, error) { @@ -77,9 +79,6 @@ func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descr // TODO: We need documentation to Library that they need to use the ocispec package for map keys // TODO: ?! We need the package.build architecutre rather than the o.targetPlatform.Arch. How do we make this clear to users? func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annotations map[string]string, arch string, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { - if annotations[ocispec.AnnotationTitle] == "" { - return errors.New("invalid annotations: please include ocispec.AnnotationTitle") - } copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency // assumes referrers API is not supported since OCI artifact @@ -93,15 +92,11 @@ func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annota if err != nil { return err } - root, err := o.generatePackManifest(src, desc, &manifestConfigDesc, annotations) + root, err := o.generatePackManifest(src, desc, manifestConfigDesc, annotations) if err != nil { return err } - // ?! Do we care about the size of these? - // Maybe we should add a total function or simliar to set the byte amount in the progress bar - // total += root.Size + manifestConfigDesc.Size - o.Transport.ProgressBar = progressBar publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), o.repo, "", copyOpts) @@ -109,7 +104,7 @@ func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annota return err } - if err := o.UpdateIndex(o.repo.Reference.Reference, arch, publishedDesc); err != nil { + if err := o.UpdateIndex(o.repo.Reference.Reference, publishedDesc); err != nil { return err } @@ -117,18 +112,13 @@ func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annota } // UpdateIndex updates the index for the given package. -func (o *OrasRemote) UpdateIndex(tag string, arch string, publishedDesc ocispec.Descriptor) error { +func (o *OrasRemote) UpdateIndex(tag string, publishedDesc ocispec.Descriptor) error { var index ocispec.Index o.repo.Reference.Reference = tag // since ref has changed, need to reset root o.root = nil - platform := &ocispec.Platform{ - OS: MultiOS, - Architecture: arch, - } - _, err := o.repo.Resolve(o.ctx, o.repo.Reference.Reference) if err != nil { if errors.Is(err, errdef.ErrNotFound) { @@ -141,7 +131,7 @@ func (o *OrasRemote) UpdateIndex(tag string, arch string, publishedDesc ocispec. MediaType: ocispec.MediaTypeImageManifest, Digest: publishedDesc.Digest, Size: publishedDesc.Size, - Platform: platform, + Platform: o.targetPlatform, }, }, } @@ -167,10 +157,10 @@ func (o *OrasRemote) UpdateIndex(tag string, arch string, publishedDesc ocispec. found := false for idx, m := range index.Manifests { - if m.Platform != nil && m.Platform.Architecture == arch { + if m.Platform != nil && m.Platform.Architecture == o.targetPlatform.Architecture { index.Manifests[idx].Digest = publishedDesc.Digest index.Manifests[idx].Size = publishedDesc.Size - index.Manifests[idx].Platform = platform + index.Manifests[idx].Platform = o.targetPlatform found = true break } @@ -180,7 +170,7 @@ func (o *OrasRemote) UpdateIndex(tag string, arch string, publishedDesc ocispec. MediaType: ocispec.MediaTypeImageManifest, Digest: publishedDesc.Digest, Size: publishedDesc.Size, - Platform: platform, + Platform: o.targetPlatform, }) } diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index 3b7eeab5d3..3c53a8f317 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -13,7 +13,8 @@ type ZarfOrasRemote struct { type Modifier func(*oci.OrasRemote) func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { - remote, err := oci.NewOrasRemote(url, message.Infof, platform, mod...) + modifiers := append(mod, oci.WithMediaType("")) + remote, err := oci.NewOrasRemote(url, message.Infof, platform, modifiers...) if err != nil { return nil, err } diff --git a/src/pkg/ocizarf/push.go b/src/pkg/ocizarf/push.go index 4e4467b3d9..740c777db8 100644 --- a/src/pkg/ocizarf/push.go +++ b/src/pkg/ocizarf/push.go @@ -51,7 +51,6 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf total += desc.Size } - message.Infof("total is %d", total) progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) annotations := zarfPackageOciAnnotations(&pkg.Metadata) err = o.PublishPackage(ctx, src, annotations, pkg.Build.Architecture, descs, config.CommonOptions.OCIConcurrency, progressBar) @@ -60,7 +59,6 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf return fmt.Errorf("unable to publish package: %w", err) } - // ?! Do I know the media type 100% at this point progressBar.Successf("Published %s [%s]", o.Repo().Reference, oci.ZarfLayerMediaTypeBlob) return nil } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 33aea59831..ea800ff60e 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -77,7 +77,7 @@ func (p *Packager) Publish() (err error) { } tag := srcRemote.Repo().Reference.Reference - if err := dstRemote.UpdateIndex(tag, arch, expected); err != nil { + if err := dstRemote.UpdateIndex(tag, expected); err != nil { return err } message.Infof("Published %s to %s", srcRemote.Repo().Reference, dstRemote.Repo().Reference) @@ -112,8 +112,13 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } - - remote, err := ocizarf.NewZarfOrasRemote(ref, oci.PlatformForArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) + var platform ocispec.Platform + if p.cfg.CreateOpts.IsSkeleton { + platform = oci.PlatformForSkeleton() + } else { + platform = oci.PlatformForArch(config.GetArch()) + } + remote, err := ocizarf.NewZarfOrasRemote(ref, platform, oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return err } From 7655d32f3aea5b55caa3f3c9e7545cbb4673cb3a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 30 Jan 2024 20:17:37 +0000 Subject: [PATCH 021/113] add media type --- src/pkg/ocizarf/common.go | 2 +- src/pkg/ocizarf/push.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index 3c53a8f317..f1f76962ba 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -13,7 +13,7 @@ type ZarfOrasRemote struct { type Modifier func(*oci.OrasRemote) func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { - modifiers := append(mod, oci.WithMediaType("")) + modifiers := append(mod, oci.WithMediaType(oci.ZarfConfigMediaType)) remote, err := oci.NewOrasRemote(url, message.Infof, platform, modifiers...) if err != nil { return nil, err diff --git a/src/pkg/ocizarf/push.go b/src/pkg/ocizarf/push.go index 740c777db8..54367a0ef2 100644 --- a/src/pkg/ocizarf/push.go +++ b/src/pkg/ocizarf/push.go @@ -68,7 +68,6 @@ func zarfPackageOciAnnotations(metadata *types.ZarfMetadata) map[string]string { annotations := map[string]string{ ocispec.AnnotationTitle: metadata.Name, ocispec.AnnotationDescription: metadata.Description, - ocispec.MediaTypeImageConfig: oci.ZarfConfigMediaType, } if url := metadata.URL; url != "" { From 9fc13dbd4770312ee092bae2494afaf15151db5e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 30 Jan 2024 20:24:39 +0000 Subject: [PATCH 022/113] move transport to helpers --- src/internal/packager/images/push.go | 3 +- src/pkg/oci/common.go | 10 +-- src/pkg/ocizarf/common.go | 7 +- src/pkg/utils/transport.go | 103 --------------------------- 4 files changed, 11 insertions(+), 112 deletions(-) delete mode 100644 src/pkg/utils/transport.go diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index bc0e3aa4a8..6f4fa08a11 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/logs" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -52,7 +53,7 @@ func (i *ImageConfig) PushToZarfRegistry() error { httpTransport.TLSClientConfig.InsecureSkipVerify = i.Insecure progressBar := message.NewProgressBar(totalSize, fmt.Sprintf("Pushing %d images to the zarf registry", len(i.ImageList))) defer progressBar.Stop() - craneTransport := utils.NewTransport(httpTransport, progressBar) + craneTransport := helpers.NewTransport(httpTransport, progressBar) pushOptions := config.GetCraneOptions(i.Insecure, i.Architectures...) pushOptions = append(pushOptions, config.GetCraneAuthOption(i.RegInfo.PushUsername, i.RegInfo.PushPassword)) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 6e83c0a90e..7e341ec70f 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -11,7 +11,6 @@ import ( "net/http" "strings" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" @@ -25,8 +24,7 @@ import ( const ( // ZarfLayerMediaTypeBlob is the media type for all Zarf layers due to the range of possible content ZarfLayerMediaTypeBlob = "application/vnd.zarf.layer.v1.blob" - // ZarfConfigMediaType is the media type for the manifest config - ZarfConfigMediaType = "application/vnd.zarf.config.v1+json" + // SkeletonArch is the architecture used for skeleton packages SkeletonArch = "skeleton" // MultiOS is the OS used for multi-platform packages @@ -55,7 +53,7 @@ type OrasRemote struct { repo *remote.Repository root *OCIManifest ctx context.Context - Transport *utils.Transport + Transport *helpers.Transport CopyOpts oras.CopyOptions targetPlatform *ocispec.Platform userAgent string @@ -136,8 +134,6 @@ func WithUserAgent(userAgent string) Modifier { // NewOrasRemote returns an oras remote repository client and context for the given url. // // # Registry auth is handled by the Docker CLI's credential store and checked before returning the client -// -// ?! Should I keep platform in this function or only have it in NewZarfOrasRemote? func NewOrasRemote(url string, logger log, platform ocispec.Platform, mods ...Modifier) (*OrasRemote, error) { ref, err := registry.ParseReference(strings.TrimPrefix(url, helpers.OCIURLPrefix)) if err != nil { @@ -208,7 +204,7 @@ func (o *OrasRemote) setRepository(ref registry.Reference) error { // TODO: instead of using Docker's cred store, should use the new one from ORAS to remove that dep func (o *OrasRemote) createAuthClient(ref registry.Reference) (*auth.Client, error) { transport := http.DefaultTransport.(*http.Transport).Clone() - o.Transport = utils.NewTransport(transport, nil) + o.Transport = helpers.NewTransport(transport, nil) client := &auth.Client{ Cache: auth.DefaultCache, diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index f1f76962ba..d62c8cf948 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -6,6 +6,11 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) +var ( + // ZarfConfigMediaType is the media type for the manifest config + ZarfConfigMediaType = "application/vnd.zarf.config.v1+json" +) + type ZarfOrasRemote struct { *oci.OrasRemote } @@ -13,7 +18,7 @@ type ZarfOrasRemote struct { type Modifier func(*oci.OrasRemote) func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { - modifiers := append(mod, oci.WithMediaType(oci.ZarfConfigMediaType)) + modifiers := append(mod, oci.WithMediaType(ZarfConfigMediaType)) remote, err := oci.NewOrasRemote(url, message.Infof, platform, modifiers...) if err != nil { return nil, err diff --git a/src/pkg/utils/transport.go b/src/pkg/utils/transport.go deleted file mode 100644 index 85d5a5c65d..0000000000 --- a/src/pkg/utils/transport.go +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package utils provides generic utility functions. -package utils - -import ( - "io" - "net/http" - "time" - - "oras.land/oras-go/v2/registry/remote/retry" -) - -type progressWriter interface { - UpdateTitle(string) - io.Writer -} - -// Transport is an http.RoundTripper that keeps track of the in-flight -// request and add hooks to report upload progress. -type Transport struct { - Base http.RoundTripper - ProgressBar progressWriter -} - -// NewTransport returns a custom transport that tracks an http.RoundTripper and a message.ProgressBar. -func NewTransport(base http.RoundTripper, bar progressWriter) *Transport { - return &Transport{ - Base: base, - ProgressBar: bar, - } -} - -// RoundTrip is mirrored from retry, but instead of calling retry's private t.roundTrip(), this uses -// our own which has interactions w/ message.ProgressBar -// -// https://github.com/oras-project/oras-go/blob/main/registry/remote/retry/client.go -func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { - ctx := req.Context() - policy := retry.DefaultPolicy - attempt := 0 - for { - resp, respErr := t.roundTrip(req) - duration, err := policy.Retry(attempt, resp, respErr) - if err != nil { - if respErr == nil { - resp.Body.Close() - } - return nil, err - } - if duration < 0 { - return resp, respErr - } - - // rewind the body if possible - if req.Body != nil { - if req.GetBody == nil { - // body can't be rewound, so we can't retry - return resp, respErr - } - body, err := req.GetBody() - if err != nil { - // failed to rewind the body, so we can't retry - return resp, respErr - } - req.Body = body - } - - // close the response body if needed - if respErr == nil { - resp.Body.Close() - } - - timer := time.NewTimer(duration) - select { - case <-ctx.Done(): - timer.Stop() - return nil, ctx.Err() - case <-timer.C: - } - attempt++ - } -} - -// roundTrip calls base roundtrip while keeping track of the current request. -// this is currently only used to track the progress of publishes, not pulls. -func (t *Transport) roundTrip(req *http.Request) (resp *http.Response, err error) { - if req.Method != http.MethodHead && req.Body != nil && t.ProgressBar != nil { - req.Body = io.NopCloser(io.TeeReader(req.Body, t.ProgressBar)) - } - - resp, err = t.Base.RoundTrip(req) - - if resp != nil && req.Method == http.MethodHead && err == nil && t.ProgressBar != nil { - if resp.ContentLength > 0 { - contentLength := int(resp.ContentLength) - data := make([]byte, contentLength) - t.ProgressBar.Write(data) - } - } - return resp, err -} From 289f02dbac7af6b5589bedcf0b76b6fe88a4a6c9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 30 Jan 2024 20:24:58 +0000 Subject: [PATCH 023/113] move transport to helpers actually --- src/pkg/utils/helpers/transport.go | 103 +++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 src/pkg/utils/helpers/transport.go diff --git a/src/pkg/utils/helpers/transport.go b/src/pkg/utils/helpers/transport.go new file mode 100644 index 0000000000..3172877d40 --- /dev/null +++ b/src/pkg/utils/helpers/transport.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package utils provides generic utility functions. +package helpers + +import ( + "io" + "net/http" + "time" + + "oras.land/oras-go/v2/registry/remote/retry" +) + +type progressWriter interface { + UpdateTitle(string) + io.Writer +} + +// Transport is an http.RoundTripper that keeps track of the in-flight +// request and add hooks to report upload progress. +type Transport struct { + Base http.RoundTripper + ProgressBar progressWriter +} + +// NewTransport returns a custom transport that tracks an http.RoundTripper and a message.ProgressBar. +func NewTransport(base http.RoundTripper, bar progressWriter) *Transport { + return &Transport{ + Base: base, + ProgressBar: bar, + } +} + +// RoundTrip is mirrored from retry, but instead of calling retry's private t.roundTrip(), this uses +// our own which has interactions w/ message.ProgressBar +// +// https://github.com/oras-project/oras-go/blob/main/registry/remote/retry/client.go +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + ctx := req.Context() + policy := retry.DefaultPolicy + attempt := 0 + for { + resp, respErr := t.roundTrip(req) + duration, err := policy.Retry(attempt, resp, respErr) + if err != nil { + if respErr == nil { + resp.Body.Close() + } + return nil, err + } + if duration < 0 { + return resp, respErr + } + + // rewind the body if possible + if req.Body != nil { + if req.GetBody == nil { + // body can't be rewound, so we can't retry + return resp, respErr + } + body, err := req.GetBody() + if err != nil { + // failed to rewind the body, so we can't retry + return resp, respErr + } + req.Body = body + } + + // close the response body if needed + if respErr == nil { + resp.Body.Close() + } + + timer := time.NewTimer(duration) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + attempt++ + } +} + +// roundTrip calls base roundtrip while keeping track of the current request. +// this is currently only used to track the progress of publishes, not pulls. +func (t *Transport) roundTrip(req *http.Request) (resp *http.Response, err error) { + if req.Method != http.MethodHead && req.Body != nil && t.ProgressBar != nil { + req.Body = io.NopCloser(io.TeeReader(req.Body, t.ProgressBar)) + } + + resp, err = t.Base.RoundTrip(req) + + if resp != nil && req.Method == http.MethodHead && err == nil && t.ProgressBar != nil { + if resp.ContentLength > 0 { + contentLength := int(resp.ContentLength) + data := make([]byte, contentLength) + t.ProgressBar.Write(data) + } + } + return resp, err +} From 41a83dc376c8129ac2fb460841747b1b3f6b1fe0 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 31 Jan 2024 15:33:24 +0000 Subject: [PATCH 024/113] WIP pulling progressbar out of render, breaking skeletons on purpose --- src/pkg/oci/pull.go | 16 ++++++---------- src/pkg/oci/utils.go | 8 ++++++++ src/pkg/ocizarf/pull.go | 14 +++++++++++++- src/pkg/packager/composer/oci.go | 8 ++++---- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 7734efa6cc..9583829590 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -53,7 +53,8 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi // PullLayers pulls the package from the remote repository and saves it to the given path. // // layersToPull is an optional parameter that allows the caller to specify which layers to pull. -func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, layersToPull []ocispec.Descriptor) ([]ocispec.Descriptor, error) { +func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, + layersToPull []ocispec.Descriptor, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) ([]ocispec.Descriptor, error) { // de-duplicate layers layersToPull = RemoveDuplicateDescriptors(layersToPull) @@ -66,11 +67,12 @@ func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, layersTo copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, destinationDir) + return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, destinationDir, doneSaving, encounteredErr, wg) } // CopyWithProgress copies the given layers from the remote repository to the given store. -func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, copyOpts oras.CopyOptions, destinationDir string) error { +func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, + copyOpts oras.CopyOptions, destinationDir string, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) error { estimatedBytes := int64(0) shas := []string{} for _, layer := range layers { @@ -110,13 +112,6 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta } } - // Create a thread to update a progress bar as we save the package to disk - doneSaving := make(chan int) - encounteredErr := make(chan int) - var wg sync.WaitGroup - wg.Add(1) - successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+o.repo.Reference.String()) - go utils.RenderProgressBarForLocalDirWrite(destinationDir, estimatedBytes, &wg, doneSaving, encounteredErr, "Pulling", successText) _, err := oras.Copy(o.ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) if err != nil { encounteredErr <- 1 @@ -131,6 +126,7 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta } // PullLayer pulls a layer from the remote repository and saves it to `destinationDir/annotationTitle`. +// ?! Why do we pull a single layer with o.fetchLayer, but multiple layers with oras.copy func (o *OrasRemote) PullLayer(desc ocispec.Descriptor, destinationDir string) error { // ?! Would this not fail in a different way if this error wasn't here // Is this simply for better error messaging? diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index 20e541ebf6..c240922f66 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -43,3 +43,11 @@ func RemoveDuplicateDescriptors(descriptors []ocispec.Descriptor) []ocispec.Desc } return list } + +func SumLayersSize(layers []ocispec.Descriptor) int64 { + var sum int64 + for _, layer := range layers { + sum += layer.Size + } + return sum +} diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index 7b1e77ef1d..1791e0e865 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -8,11 +8,13 @@ import ( "fmt" "path/filepath" "slices" + "sync" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -56,7 +58,17 @@ func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, lay } layersToPull = append(layersToPull, manifest.Config) - return o.PullLayers(destinationDir, concurrency, layersToPull) + // Create a thread to update a progress bar as we save the package to disk + doneSaving := make(chan int) + encounteredErr := make(chan int) + var wg sync.WaitGroup + wg.Add(1) + successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+o.Repo().Reference.String()) + + layerSize := oci.SumLayersSize(layersToPull) + go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) + + return o.PullLayers(destinationDir, concurrency, layersToPull, doneSaving, encounteredErr, &wg) } // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 02562b3254..abf5ef3818 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -18,7 +18,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/mholt/archiver/v3" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/content" ocistore "oras.land/oras-go/v2/content/oci" ) @@ -99,9 +98,10 @@ func (ic *ImportChain) fetchOCISkeleton() error { copyOpts := remote.CopyOpts // TODO (@WSTARR): This overrides the FindSuccessors function to no longer filter nodes when pulling which is necessary when caching - once we implement caching more thoroughly we will need to reevaluate this. copyOpts.FindSuccessors = content.Successors - if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache); err != nil { - return err - } + return err + // if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache); err != nil { + // return err + // } } } From 9d3ecb354dc7474124b59bf6417bcea42d79e7f1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 31 Jan 2024 15:51:48 +0000 Subject: [PATCH 025/113] WIP inital use of concurrency tools --- src/pkg/oci/pull.go | 10 +++++----- src/pkg/ocizarf/pull.go | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 9583829590..212d6dfe3d 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -54,7 +54,7 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi // // layersToPull is an optional parameter that allows the caller to specify which layers to pull. func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, - layersToPull []ocispec.Descriptor, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) ([]ocispec.Descriptor, error) { + layersToPull []ocispec.Descriptor, ct *helpers.ConcurrencyTools[int, int], wg *sync.WaitGroup) ([]ocispec.Descriptor, error) { // de-duplicate layers layersToPull = RemoveDuplicateDescriptors(layersToPull) @@ -67,12 +67,12 @@ func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, destinationDir, doneSaving, encounteredErr, wg) + return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, destinationDir, ct, wg) } // CopyWithProgress copies the given layers from the remote repository to the given store. func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, - copyOpts oras.CopyOptions, destinationDir string, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) error { + copyOpts oras.CopyOptions, destinationDir string, ct *helpers.ConcurrencyTools[int, int], wg *sync.WaitGroup) error { estimatedBytes := int64(0) shas := []string{} for _, layer := range layers { @@ -114,12 +114,12 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta _, err := oras.Copy(o.ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) if err != nil { - encounteredErr <- 1 + ct.ErrorChan <- 1 return err } // Send a signal to the progress bar that we're done and wait for it to finish - doneSaving <- 1 + ct.ProgressChan <- 1 wg.Wait() return nil diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index 1791e0e865..e4193c1227 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -59,16 +59,16 @@ func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, lay layersToPull = append(layersToPull, manifest.Config) // Create a thread to update a progress bar as we save the package to disk - doneSaving := make(chan int) - encounteredErr := make(chan int) + // Maybe the length should be len() + ct := helpers.NewConcurrencyTools[int, int](len(layersToPull)) var wg sync.WaitGroup wg.Add(1) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+o.Repo().Reference.String()) layerSize := oci.SumLayersSize(layersToPull) - go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) + go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, &wg, ct.ProgressChan, ct.ErrorChan, "Pulling", successText) - return o.PullLayers(destinationDir, concurrency, layersToPull, doneSaving, encounteredErr, &wg) + return o.PullLayers(destinationDir, concurrency, layersToPull, ct, &wg) } // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. From 0e0f5a5bd6faad7bb98205e8858ea0568e07443d Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 31 Jan 2024 17:55:32 +0000 Subject: [PATCH 026/113] WIP, tests passing creating io in helpers --- src/extensions/bigbang/bigbang.go | 2 +- src/internal/packager/helm/post-render.go | 3 ++- src/internal/packager/kustomize/build.go | 4 +-- src/pkg/cluster/data.go | 3 ++- src/pkg/oci/pull.go | 19 +++++--------- src/pkg/oci/push.go | 2 -- src/pkg/ocizarf/pull.go | 7 ++--- src/pkg/packager/composer/oci.go | 17 ++++++++++--- src/pkg/packager/create_stages.go | 2 +- src/pkg/utils/helpers/io.go | 31 +++++++++++++++++++++++ src/pkg/utils/io.go | 21 --------------- src/test/e2e/20_zarf_init_test.go | 4 +-- 12 files changed, 64 insertions(+), 51 deletions(-) create mode 100644 src/pkg/utils/helpers/io.go diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 66427a682f..571f886e25 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -465,7 +465,7 @@ func addBigBangManifests(YOLO bool, manifestDir string, cfg *extensions.BigBang) return err } - if err := utils.WriteFile(path, out); err != nil { + if err := helpers.WriteFile(path, out); err != nil { return err } diff --git a/src/internal/packager/helm/post-render.go b/src/internal/packager/helm/post-render.go index 7d89098ad3..60c4aa042b 100644 --- a/src/internal/packager/helm/post-render.go +++ b/src/internal/packager/helm/post-render.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/releaseutil" @@ -62,7 +63,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { path := filepath.Join(tempDir, "chart.yaml") // Write the context to a file for processing - if err := utils.WriteFile(path, renderedManifests.Bytes()); err != nil { + if err := helpers.WriteFile(path, renderedManifests.Bytes()); err != nil { return nil, fmt.Errorf("unable to write the post-render file for the helm chart") } diff --git a/src/internal/packager/kustomize/build.go b/src/internal/packager/kustomize/build.go index e1481ff8ec..9b52c8e629 100644 --- a/src/internal/packager/kustomize/build.go +++ b/src/internal/packager/kustomize/build.go @@ -7,7 +7,7 @@ package kustomize import ( "fmt" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "sigs.k8s.io/kustomize/api/krusty" krustytypes "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/filesys" @@ -39,5 +39,5 @@ func Build(path string, destination string, kustomizeAllowAnyDirectory bool) err return fmt.Errorf("problem converting kustomization to yaml: %w", err) } - return utils.WriteFile(destination, yaml) + return helpers.WriteFile(destination, yaml) } diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index 3ce8752e4f..7eb57fa6a3 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" corev1 "k8s.io/api/core/v1" ) @@ -28,7 +29,7 @@ func (c *Cluster) HandleDataInjection(wg *sync.WaitGroup, data types.ZarfDataInj defer wg.Done() injectionCompletionMarker := filepath.Join(componentPath.DataInjections, config.GetDataInjectionMarker()) - if err := utils.WriteFile(injectionCompletionMarker, []byte("🦄")); err != nil { + if err := helpers.WriteFile(injectionCompletionMarker, []byte("🦄")); err != nil { message.WarnErrf(err, "Unable to create the data injection completion marker") return } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 212d6dfe3d..80ad1097f7 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -6,14 +6,12 @@ package oci import ( "context" - "fmt" "os" "path/filepath" "sync" "slices" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" @@ -54,7 +52,7 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi // // layersToPull is an optional parameter that allows the caller to specify which layers to pull. func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, - layersToPull []ocispec.Descriptor, ct *helpers.ConcurrencyTools[int, int], wg *sync.WaitGroup) ([]ocispec.Descriptor, error) { + layersToPull []ocispec.Descriptor, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) ([]ocispec.Descriptor, error) { // de-duplicate layers layersToPull = RemoveDuplicateDescriptors(layersToPull) @@ -67,12 +65,12 @@ func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, destinationDir, ct, wg) + return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, destinationDir, doneSaving, encounteredErr, wg) } // CopyWithProgress copies the given layers from the remote repository to the given store. func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, - copyOpts oras.CopyOptions, destinationDir string, ct *helpers.ConcurrencyTools[int, int], wg *sync.WaitGroup) error { + copyOpts oras.CopyOptions, destinationDir string, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) error { estimatedBytes := int64(0) shas := []string{} for _, layer := range layers { @@ -114,12 +112,12 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta _, err := oras.Copy(o.ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) if err != nil { - ct.ErrorChan <- 1 + encounteredErr <- 1 return err } // Send a signal to the progress bar that we're done and wait for it to finish - ct.ProgressChan <- 1 + doneSaving <- 1 wg.Wait() return nil @@ -128,11 +126,6 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta // PullLayer pulls a layer from the remote repository and saves it to `destinationDir/annotationTitle`. // ?! Why do we pull a single layer with o.fetchLayer, but multiple layers with oras.copy func (o *OrasRemote) PullLayer(desc ocispec.Descriptor, destinationDir string) error { - // ?! Would this not fail in a different way if this error wasn't here - // Is this simply for better error messaging? - if desc.MediaType != ZarfLayerMediaTypeBlob { - return fmt.Errorf("invalid media type for file layer: %s", desc.MediaType) - } b, err := o.FetchLayer(desc) if err != nil { return err @@ -140,7 +133,7 @@ func (o *OrasRemote) PullLayer(desc ocispec.Descriptor, destinationDir string) e rel := desc.Annotations[ocispec.AnnotationTitle] - return utils.WriteFile(filepath.Join(destinationDir, rel), b) + return helpers.WriteFile(filepath.Join(destinationDir, rel), b) } // PullPackagePaths pulls multiple files from the remote repository and saves them to `destinationDir`. diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index aa0ff09e15..9893616f16 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -76,8 +76,6 @@ func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descr } // PublishPackage publishes the package to the remote repository. -// TODO: We need documentation to Library that they need to use the ocispec package for map keys -// TODO: ?! We need the package.build architecutre rather than the o.targetPlatform.Arch. How do we make this clear to users? func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annotations map[string]string, arch string, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index e4193c1227..d01b129fa5 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -60,15 +60,16 @@ func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, lay // Create a thread to update a progress bar as we save the package to disk // Maybe the length should be len() - ct := helpers.NewConcurrencyTools[int, int](len(layersToPull)) + doneSaving := make(chan int) + encounteredErr := make(chan int) var wg sync.WaitGroup wg.Add(1) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+o.Repo().Reference.String()) layerSize := oci.SumLayersSize(layersToPull) - go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, &wg, ct.ProgressChan, ct.ErrorChan, "Pulling", successText) + go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) - return o.PullLayers(destinationDir, concurrency, layersToPull, ct, &wg) + return o.PullLayers(destinationDir, concurrency, layersToPull, doneSaving, encounteredErr, &wg) } // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index abf5ef3818..4ae54e7728 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "path/filepath" + "sync" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" @@ -17,7 +18,9 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/mholt/archiver/v3" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/content" ocistore "oras.land/oras-go/v2/content/oci" ) @@ -98,10 +101,16 @@ func (ic *ImportChain) fetchOCISkeleton() error { copyOpts := remote.CopyOpts // TODO (@WSTARR): This overrides the FindSuccessors function to no longer filter nodes when pulling which is necessary when caching - once we implement caching more thoroughly we will need to reevaluate this. copyOpts.FindSuccessors = content.Successors - return err - // if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache); err != nil { - // return err - // } + doneSaving := make(chan int) + encounteredErr := make(chan int) + var wg sync.WaitGroup + wg.Add(1) + successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) + layerSize := oci.SumLayersSize([]ocispec.Descriptor{componentDesc}) + go utils.RenderProgressBarForLocalDirWrite(cache, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) + if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache, doneSaving, encounteredErr, &wg); err != nil { + return err + } } } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 16712d4292..709d5129e6 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -637,7 +637,7 @@ func (p *Packager) generatePackageChecksums() (string, error) { // Create the checksums file checksumsFilePath := p.layout.Checksums - if err := utils.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { + if err := helpers.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { return "", err } diff --git a/src/pkg/utils/helpers/io.go b/src/pkg/utils/helpers/io.go new file mode 100644 index 0000000000..670054fa45 --- /dev/null +++ b/src/pkg/utils/helpers/io.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package helpers provides generic helper functions with no external imports +package helpers + +import ( + "fmt" + "os" +) + +// WriteFile writes the given data to the given path. +func WriteFile(path string, data []byte) error { + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("unable to create the file at %s to write the contents: %w", path, err) + } + + _, err = f.Write(data) + if err != nil { + _ = f.Close() + return fmt.Errorf("unable to write the file at %s contents:%w", path, err) + } + + err = f.Close() + if err != nil { + return fmt.Errorf("error saving file %s: %w", path, err) + } + + return nil +} diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index c02ce6b55e..c196fee45d 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -100,27 +100,6 @@ func ListDirectories(directory string) ([]string, error) { return directories, nil } -// WriteFile writes the given data to the given path. -func WriteFile(path string, data []byte) error { - f, err := os.Create(path) - if err != nil { - return fmt.Errorf("unable to create the file at %s to write the contents: %w", path, err) - } - - _, err = f.Write(data) - if err != nil { - _ = f.Close() - return fmt.Errorf("unable to write the file at %s contents:%w", path, err) - } - - err = f.Close() - if err != nil { - return fmt.Errorf("error saving file %s: %w", path, err) - } - - return nil -} - // ReplaceTextTemplate loads a file from a given path, replaces text in it and writes it back in place. func ReplaceTextTemplate(path string, mappings map[string]*TextTemplate, deprecations map[string]string, templateRegex string) error { textFile, err := os.Open(path) diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index 86f127fc83..cc94ed643f 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -12,7 +12,7 @@ import ( "encoding/json" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" ) @@ -150,7 +150,7 @@ func initWithoutStorageClass(t *testing.T) { storageClassFileName := "storage-class.yaml" - err = utils.WriteFile(storageClassFileName, []byte(storageClassYaml)) + err = helpers.WriteFile(storageClassFileName, []byte(storageClassYaml)) require.NoError(t, err) defer e2e.CleanFiles(storageClassFileName) From dac2b738276a1dcf289428b80227980b6e9eab2f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 31 Jan 2024 18:08:46 +0000 Subject: [PATCH 027/113] WIP, de-zarfifying --- src/pkg/oci/common.go | 15 +-------------- src/pkg/ocizarf/common.go | 12 ++++++++++++ src/pkg/ocizarf/pull.go | 2 +- src/pkg/ocizarf/push.go | 5 ++--- src/pkg/packager/composer/oci.go | 2 +- src/pkg/packager/publish.go | 2 +- src/pkg/packager/sources/oci.go | 3 +-- 7 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 7e341ec70f..ccbddd640b 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -22,11 +22,6 @@ import ( ) const ( - // ZarfLayerMediaTypeBlob is the media type for all Zarf layers due to the range of possible content - ZarfLayerMediaTypeBlob = "application/vnd.zarf.layer.v1.blob" - - // SkeletonArch is the architecture used for skeleton packages - SkeletonArch = "skeleton" // MultiOS is the OS used for multi-platform packages MultiOS = "multi" ) @@ -101,14 +96,6 @@ func WithInsecureSkipVerify(insecure bool) Modifier { } } -// PlatformForSkeleton returns a skeleton oci -func PlatformForSkeleton() ocispec.Platform { - return ocispec.Platform{ - OS: MultiOS, - Architecture: SkeletonArch, - } -} - // WithMediaType sets the mediatype for the remote func WithMediaType(mediaType string) Modifier { return func(o *OrasRemote) { @@ -222,7 +209,7 @@ func (o *OrasRemote) createAuthClient(ref registry.Reference) (*auth.Client, err return nil, err } if !cfg.ContainsAuth() { - o.log("no docker config file found, run 'zarf tools registry login --help'") + o.log("no docker config file found") return client, nil } diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index d62c8cf948..e08a53809a 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -9,6 +9,10 @@ import ( var ( // ZarfConfigMediaType is the media type for the manifest config ZarfConfigMediaType = "application/vnd.zarf.config.v1+json" + // ZarfLayerMediaTypeBlob is the media type for all Zarf layers due to the range of possible content + ZarfLayerMediaTypeBlob = "application/vnd.zarf.layer.v1.blob" + // SkeletonArch is the architecture used for skeleton packages + SkeletonArch = "skeleton" ) type ZarfOrasRemote struct { @@ -25,3 +29,11 @@ func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifie } return &ZarfOrasRemote{remote}, nil } + +// PlatformForSkeleton returns a skeleton oci +func PlatformForSkeleton() ocispec.Platform { + return ocispec.Platform{ + OS: oci.MultiOS, + Architecture: SkeletonArch, + } +} diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index d01b129fa5..375cf5af0f 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -134,7 +134,7 @@ func LayersFromRequestedComponents(o *ZarfOrasRemote, requestedComponents []stri }) // even though these are technically image manifests, we store them as Zarf blobs - manifestDescriptor.MediaType = oci.ZarfLayerMediaTypeBlob + manifestDescriptor.MediaType = ZarfLayerMediaTypeBlob manifest, err := o.FetchManifest(manifestDescriptor) if err != nil { diff --git a/src/pkg/ocizarf/push.go b/src/pkg/ocizarf/push.go index 54367a0ef2..c6e724732f 100644 --- a/src/pkg/ocizarf/push.go +++ b/src/pkg/ocizarf/push.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -34,7 +33,7 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf for name, path := range paths.Files() { spinner.Updatef("Preparing layer %s", helpers.First30last30(name)) - mediaType := oci.ZarfLayerMediaTypeBlob + mediaType := ZarfLayerMediaTypeBlob desc, err := src.Add(ctx, name, mediaType, path) if err != nil { @@ -59,7 +58,7 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf return fmt.Errorf("unable to publish package: %w", err) } - progressBar.Successf("Published %s [%s]", o.Repo().Reference, oci.ZarfLayerMediaTypeBlob) + progressBar.Successf("Published %s [%s]", o.Repo().Reference, ZarfLayerMediaTypeBlob) return nil } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 4ae54e7728..ef30b3f97d 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -30,7 +30,7 @@ func (ic *ImportChain) getRemote(url string) (*ocizarf.ZarfOrasRemote, error) { return ic.remote, nil } var err error - ic.remote, err = ocizarf.NewZarfOrasRemote(url, oci.PlatformForSkeleton(), oci.WithInsecure(config.CommonOptions.Insecure)) + ic.remote, err = ocizarf.NewZarfOrasRemote(url, ocizarf.PlatformForSkeleton(), oci.WithInsecure(config.CommonOptions.Insecure)) if err != nil { return nil, err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index ea800ff60e..addbcfbd27 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -114,7 +114,7 @@ func (p *Packager) Publish() (err error) { } var platform ocispec.Platform if p.cfg.CreateOpts.IsSkeleton { - platform = oci.PlatformForSkeleton() + platform = ocizarf.PlatformForSkeleton() } else { platform = oci.PlatformForArch(config.GetArch()) } diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index dc21c6a270..833737c02c 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -14,7 +14,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -200,7 +199,7 @@ func (s *OCISource) Collect(dir string) (string, error) { spinner.Success() // TODO (@Noxsios) remove the suffix check at v1.0.0 - isSkeleton := pkg.Build.Architecture == "skeleton" || strings.HasSuffix(s.Repo().Reference.Reference, oci.SkeletonArch) + isSkeleton := pkg.Build.Architecture == "skeleton" || strings.HasSuffix(s.Repo().Reference.Reference, ocizarf.SkeletonArch) name := NameFromMetadata(&pkg, isSkeleton) dstTarball := filepath.Join(dir, name) From 4c7626f062b7553582a0519a235c7c81338b6037 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 31 Jan 2024 19:40:42 +0000 Subject: [PATCH 028/113] adding logging interface --- src/pkg/message/message.go | 16 ++++++++++++++++ src/pkg/oci/common.go | 19 ++++++++++++------- src/pkg/oci/copier.go | 8 ++++---- src/pkg/oci/progress.go | 2 +- src/pkg/ocizarf/common.go | 2 +- 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index d17baca298..db6fab7693 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -38,6 +38,21 @@ const ( TermWidth = 100 ) +type Logger struct{} + +func (l *Logger) Debug(msg string, args ...any) { + Debugf(msg, args) +} +func (l *Logger) Info(msg string, args ...any) { + Infof(msg, args) +} +func (l *Logger) Warn(msg string, args ...any) { + Warnf(msg, args) +} +func (l *Logger) Error(msg string, args ...any) { + Warnf(msg, args) +} + // NoProgress tracks whether spinner/progress bars show updates. var NoProgress bool @@ -144,6 +159,7 @@ func Debug(payload ...any) { func Debugf(format string, a ...any) { message := fmt.Sprintf(format, a...) debugPrinter(2, message) + } // ErrorWebf prints an error message and returns a web response. diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index ccbddd640b..c372058673 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -26,6 +26,14 @@ const ( MultiOS = "multi" ) +// Logger is an interface built to +type Logger interface { + Debug(msg string, args ...any) + Info(msg string, args ...any) + Warn(msg string, args ...any) + Error(msg string, args ...any) +} + func (DiscardProgressWriter) Write(p []byte) (int, error) { return len(p), nil } @@ -39,9 +47,6 @@ type ProgressWriter interface { io.Writer } -// log is a function that logs a message. -type log func(string, ...any) - // OrasRemote is a wrapper around the Oras remote repository that includes a progress bar for interactive feedback. // Do we want to start exporting fields in this struct? For example log may come in handy? type OrasRemote struct { @@ -52,7 +57,7 @@ type OrasRemote struct { CopyOpts oras.CopyOptions targetPlatform *ocispec.Platform userAgent string - log log + log Logger mediaType string } @@ -121,7 +126,7 @@ func WithUserAgent(userAgent string) Modifier { // NewOrasRemote returns an oras remote repository client and context for the given url. // // # Registry auth is handled by the Docker CLI's credential store and checked before returning the client -func NewOrasRemote(url string, logger log, platform ocispec.Platform, mods ...Modifier) (*OrasRemote, error) { +func NewOrasRemote(url string, logger Logger, platform ocispec.Platform, mods ...Modifier) (*OrasRemote, error) { ref, err := registry.ParseReference(strings.TrimPrefix(url, helpers.OCIURLPrefix)) if err != nil { return nil, fmt.Errorf("failed to parse OCI reference %q: %w", url, err) @@ -203,13 +208,13 @@ func (o *OrasRemote) createAuthClient(ref registry.Reference) (*auth.Client, err client.SetUserAgent(o.userAgent) } - o.log("Loading docker config file from default config location: %s for %s", config.Dir(), ref) + o.log.Debug("Loading docker config file from default config location: %s for %s", config.Dir(), ref) cfg, err := config.Load(config.Dir()) if err != nil { return nil, err } if !cfg.ContainsAuth() { - o.log("no docker config file found") + o.log.Debug("no docker config file found") return client, nil } diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 5d139a31ca..9d2c6b3c60 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -38,9 +38,9 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, for idx, layer := range layers { bytes, err := json.MarshalIndent(layer, "", " ") if err != nil { - src.log("ERROR marshalling json: %s", err.Error()) + src.log.Debug("ERROR marshalling json: %s", err.Error()) } - src.log("Copying layer:", string(bytes)) + src.log.Debug("Copying layer:", string(bytes)) if err := sem.Acquire(ctx, 1); err != nil { return err } @@ -51,7 +51,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return err } if exists { - src.log("Layer already exists in destination, skipping") + src.log.Debug("Layer already exists in destination, skipping") contentLength := layer.Size data := make([]byte, contentLength) progressBar.Write(data) @@ -106,7 +106,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, } duration := time.Since(start) - src.log("Copied", src.repo.Reference, "to", dst.repo.Reference, "with a concurrency of", concurrency, "and took", duration) + src.log.Debug("Copied", src.repo.Reference, "to", dst.repo.Reference, "with a concurrency of", concurrency, "and took", duration) return nil } diff --git a/src/pkg/oci/progress.go b/src/pkg/oci/progress.go index faf2cfa176..79d383ed11 100644 --- a/src/pkg/oci/progress.go +++ b/src/pkg/oci/progress.go @@ -31,6 +31,6 @@ func (o *OrasRemote) printLayer(desc ocispec.Descriptor, suffix string) error { } else { layerInfo = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) } - o.log("%s (%s)", layerInfo, suffix) + o.log.Debug("%s (%s)", layerInfo, suffix) return nil } diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index e08a53809a..e291f043f6 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -23,7 +23,7 @@ type Modifier func(*oci.OrasRemote) func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { modifiers := append(mod, oci.WithMediaType(ZarfConfigMediaType)) - remote, err := oci.NewOrasRemote(url, message.Infof, platform, modifiers...) + remote, err := oci.NewOrasRemote(url, &message.Logger{}, platform, modifiers...) if err != nil { return nil, err } From 05b03afda88fbd768bd090339ea2435aec291b51 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 31 Jan 2024 19:54:25 +0000 Subject: [PATCH 029/113] Refactor / rename --- src/pkg/oci/manifest.go | 9 --------- src/pkg/oci/pull.go | 4 ++-- src/pkg/ocizarf/copier.go | 2 +- src/pkg/ocizarf/pull.go | 4 ++-- src/pkg/packager/sources/oci.go | 2 +- 5 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index 1c2e77ba39..3406275e58 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -33,15 +33,6 @@ func (m *OCIManifest) Locate(pathOrDigest string) ocispec.Descriptor { }) } -// SumLayersSize returns the sum of the size of all the layers in the manifest. -func (m *OCIManifest) SumLayersSize() int64 { - var sum int64 - for _, layer := range m.Layers { - sum += layer.Size - } - return sum -} - // MarshalJSON returns the JSON encoding of the manifest. func (m *OCIManifest) MarshalJSON() ([]byte, error) { return json.Marshal(m.Manifest) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 80ad1097f7..17c7938a32 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -136,8 +136,8 @@ func (o *OrasRemote) PullLayer(desc ocispec.Descriptor, destinationDir string) e return helpers.WriteFile(filepath.Join(destinationDir, rel), b) } -// PullPackagePaths pulls multiple files from the remote repository and saves them to `destinationDir`. -func (o *OrasRemote) PullPackagePaths(paths []string, destinationDir string) ([]ocispec.Descriptor, error) { +// PullFilesAtPaths pulls multiple files from the remote repository and saves them to `destinationDir`. +func (o *OrasRemote) PullFilesAtPaths(paths []string, destinationDir string) ([]ocispec.Descriptor, error) { paths = helpers.Unique(paths) root, err := o.FetchRoot() if err != nil { diff --git a/src/pkg/ocizarf/copier.go b/src/pkg/ocizarf/copier.go index 6a0e88bce1..95c2b4056c 100644 --- a/src/pkg/ocizarf/copier.go +++ b/src/pkg/ocizarf/copier.go @@ -21,7 +21,7 @@ func CopyPackage(ctx context.Context, src *ZarfOrasRemote, dst *ZarfOrasRemote, return err } layers := srcRoot.GetLayers(include) - size := srcRoot.SumLayersSize() + size := oci.SumLayersSize(srcRoot.Layers) title := fmt.Sprintf("[0/%d] layers copied", len(layers)) progressBar := message.NewProgressBar(size, title) diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index 375cf5af0f..d28e9e160f 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -156,10 +156,10 @@ func LayersFromRequestedComponents(o *ZarfOrasRemote, requestedComponents []stri // PullPackageMetadata pulls the package metadata from the remote repository and saves it to `destinationDir`. func (o *ZarfOrasRemote) PullPackageMetadata(destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullPackagePaths(PackageAlwaysPull, destinationDir) + return o.PullFilesAtPaths(PackageAlwaysPull, destinationDir) } // PullPackageSBOM pulls the package's sboms.tar from the remote repository and saves it to `destinationDir`. func (o *ZarfOrasRemote) PullPackageSBOM(destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullPackagePaths([]string{layout.SBOMTar}, destinationDir) + return o.PullFilesAtPaths([]string{layout.SBOMTar}, destinationDir) } diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 833737c02c..8645b38554 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -122,7 +122,7 @@ func (s *OCISource) LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, toPull = append(toPull, layout.SBOMTar) } - layersFetched, err := s.PullPackagePaths(toPull, dst.Base) + layersFetched, err := s.PullFilesAtPaths(toPull, dst.Base) if err != nil { return err } From b8b0e6b5732e782a527633d1c12807baafe9bfd7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 31 Jan 2024 20:20:02 +0000 Subject: [PATCH 030/113] accounting for nil --- src/pkg/message/message.go | 8 ++++---- src/pkg/oci/pull.go | 17 ++++++++++++----- src/pkg/ocizarf/pull.go | 1 - 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index db6fab7693..482706c83f 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -41,16 +41,16 @@ const ( type Logger struct{} func (l *Logger) Debug(msg string, args ...any) { - Debugf(msg, args) + Debugf(msg, args...) } func (l *Logger) Info(msg string, args ...any) { - Infof(msg, args) + Infof(msg, args...) } func (l *Logger) Warn(msg string, args ...any) { - Warnf(msg, args) + Warnf(msg, args...) } func (l *Logger) Error(msg string, args ...any) { - Warnf(msg, args) + Warnf(msg, args...) } // NoProgress tracks whether spinner/progress bars show updates. diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 17c7938a32..c42b7bc2ac 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -49,8 +49,8 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi } // PullLayers pulls the package from the remote repository and saves it to the given path. -// -// layersToPull is an optional parameter that allows the caller to specify which layers to pull. +// If you don't have nil paramaters for doneSaving, encounteredErr, and wg +// you must use the channels in a go routine and call wg.done after they are used func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, layersToPull []ocispec.Descriptor, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) ([]ocispec.Descriptor, error) { // de-duplicate layers @@ -112,13 +112,20 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta _, err := oras.Copy(o.ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) if err != nil { - encounteredErr <- 1 + if encounteredErr != nil { + encounteredErr <- 1 + } return err } // Send a signal to the progress bar that we're done and wait for it to finish - doneSaving <- 1 - wg.Wait() + if doneSaving != nil { + doneSaving <- 1 + } + + if wg != nil { + wg.Wait() + } return nil } diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index d28e9e160f..5d95052529 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -59,7 +59,6 @@ func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, lay layersToPull = append(layersToPull, manifest.Config) // Create a thread to update a progress bar as we save the package to disk - // Maybe the length should be len() doneSaving := make(chan int) encounteredErr := make(chan int) var wg sync.WaitGroup From 7bd1135934339e17986eaa37552ac3d27f457271 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 31 Jan 2024 21:17:18 +0000 Subject: [PATCH 031/113] lint --- src/pkg/message/message.go | 8 ++++++++ src/pkg/message/progress.go | 5 +---- src/pkg/oci/common.go | 10 ++++++++-- src/pkg/oci/fetch.go | 10 +++++----- src/pkg/oci/manifest.go | 15 ++++++++------- src/pkg/oci/pull.go | 4 ++-- src/pkg/oci/push.go | 9 +++------ src/pkg/oci/utils.go | 5 +++-- src/pkg/ocizarf/common.go | 9 +++++++-- src/pkg/ocizarf/copier.go | 10 ++++------ src/pkg/ocizarf/fetch.go | 4 ++++ src/pkg/ocizarf/pull.go | 4 ++-- src/pkg/ocizarf/push.go | 5 +++-- src/pkg/ocizarf/utils.go | 7 ++++++- src/pkg/packager/composer/oci.go | 4 ++-- src/pkg/utils/helpers/transport.go | 2 +- 16 files changed, 67 insertions(+), 44 deletions(-) diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index 482706c83f..d311995fed 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -38,17 +38,25 @@ const ( TermWidth = 100 ) +// Logger is an object with access to the main printing functions in message type Logger struct{} +// Debug prints a debug message. func (l *Logger) Debug(msg string, args ...any) { Debugf(msg, args...) } + +// Info prints an Info message. func (l *Logger) Info(msg string, args ...any) { Infof(msg, args...) } + +// Warn prints a warning message func (l *Logger) Warn(msg string, args ...any) { Warnf(msg, args...) } + +// Error prints a warning message func (l *Logger) Error(msg string, args ...any) { Warnf(msg, args...) } diff --git a/src/pkg/message/progress.go b/src/pkg/message/progress.go index c9df5a1e24..be2da348f7 100644 --- a/src/pkg/message/progress.go +++ b/src/pkg/message/progress.go @@ -48,10 +48,6 @@ func (p *ProgressBar) Update(complete int64, text string) { p.Add(chunk) } -func (p *ProgressBar) Current() int64 { - return int64(p.progress.Current) -} - // UpdateTitle updates the ProgressBar with new text. func (p *ProgressBar) UpdateTitle(text string) { if NoProgress { @@ -82,6 +78,7 @@ func (p *ProgressBar) Write(data []byte) (int, error) { return n, nil } +// Finish stops the progressbar and logs the approiate message based on if there was an error func (p *ProgressBar) Finish(err error, format string, a ...any) { if err != nil { p.Errorf(err, format, a...) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index c372058673..982f3c40ef 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -34,24 +34,29 @@ type Logger interface { Error(msg string, args ...any) } +// Write doesn't do anything but satify implementation func (DiscardProgressWriter) Write(p []byte) (int, error) { return len(p), nil } +// UpdateTitle doesn't do anything but satify implementation func (DiscardProgressWriter) UpdateTitle(_ string) {} +// DiscardProgressWriter is a ProgressWriter in which all calls succeed without doing anything +// Use this or nil or if you don't care about writing progress type DiscardProgressWriter struct{} +// ProgressWriter wraps io.Writer, but also includes an updateTitle function to give the user +// additional context on what's going on. Useful in OCI for tracking layers type ProgressWriter interface { UpdateTitle(string) io.Writer } // OrasRemote is a wrapper around the Oras remote repository that includes a progress bar for interactive feedback. -// Do we want to start exporting fields in this struct? For example log may come in handy? type OrasRemote struct { repo *remote.Repository - root *OCIManifest + root *Manifest ctx context.Context Transport *helpers.Transport CopyOpts oras.CopyOptions @@ -71,6 +76,7 @@ func WithContext(ctx context.Context) Modifier { } } +// WithInsecure sets plainHTTP and insecure tls for the remote func WithInsecure(insecure bool) Modifier { return func(o *OrasRemote) { plainHTTPMod := WithPlainHTTP(insecure) diff --git a/src/pkg/oci/fetch.go b/src/pkg/oci/fetch.go index 0e0333b1e9..b2c8c84be0 100644 --- a/src/pkg/oci/fetch.go +++ b/src/pkg/oci/fetch.go @@ -40,7 +40,7 @@ func (o *OrasRemote) ResolveRoot() (ocispec.Descriptor, error) { } // FetchRoot fetches the root manifest from the remote repository. -func (o *OrasRemote) FetchRoot() (*OCIManifest, error) { +func (o *OrasRemote) FetchRoot() (*Manifest, error) { if o.root != nil { return o.root, nil } @@ -60,8 +60,8 @@ func (o *OrasRemote) FetchRoot() (*OCIManifest, error) { } // FetchManifest fetches the manifest with the given descriptor from the remote repository. -func (o *OrasRemote) FetchManifest(desc ocispec.Descriptor) (manifest *OCIManifest, err error) { - return FetchUnmarshal[*OCIManifest](o.FetchLayer, json.Unmarshal, desc) +func (o *OrasRemote) FetchManifest(desc ocispec.Descriptor) (manifest *Manifest, err error) { + return FetchUnmarshal[*Manifest](o.FetchLayer, json.Unmarshal, desc) } // FetchLayer fetches the layer with the given descriptor from the remote repository. @@ -70,7 +70,7 @@ func (o *OrasRemote) FetchLayer(desc ocispec.Descriptor) (bytes []byte, err erro } // FetchJSONFile fetches the given JSON file from the remote repository. -func FetchJSONFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *OCIManifest, path string) (result T, err error) { +func FetchJSONFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *Manifest, path string) (result T, err error) { descriptor := manifest.Locate(path) if IsEmptyDescriptor(descriptor) { return result, fmt.Errorf("unable to find %s in the manifest", path) @@ -79,7 +79,7 @@ func FetchJSONFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, e } // FetchYAMLFile fetches the given YAML file from the remote repository. -func FetchYAMLFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *OCIManifest, path string) (result T, err error) { +func FetchYAMLFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *Manifest, path string) (result T, err error) { descriptor := manifest.Locate(path) if IsEmptyDescriptor(descriptor) { return result, fmt.Errorf("unable to find %s in the manifest", path) diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index 3406275e58..0f6fce395e 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -12,21 +12,21 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// OCIManifest is a wrapper around the OCI manifest +// Manifest is a wrapper around the OCI manifest // // it includes the path to the index.json, oci-layout, and image blobs. // as well as a few helper functions for locating layers and calculating the size of the layers. -type OCIManifest struct { +type Manifest struct { ocispec.Manifest } // New returns a new OCIManifest -func New(manifest *ocispec.Manifest) *OCIManifest { - return &OCIManifest{*manifest} +func New(manifest *ocispec.Manifest) *Manifest { + return &Manifest{*manifest} } // Locate returns the descriptor for the first layer with the given path or digest. -func (m *OCIManifest) Locate(pathOrDigest string) ocispec.Descriptor { +func (m *Manifest) Locate(pathOrDigest string) ocispec.Descriptor { return helpers.Find(m.Layers, func(layer ocispec.Descriptor) bool { // Convert from the OS path separator to the standard '/' for Windows support return layer.Annotations[ocispec.AnnotationTitle] == filepath.ToSlash(pathOrDigest) || layer.Digest.Encoded() == pathOrDigest @@ -34,11 +34,12 @@ func (m *OCIManifest) Locate(pathOrDigest string) ocispec.Descriptor { } // MarshalJSON returns the JSON encoding of the manifest. -func (m *OCIManifest) MarshalJSON() ([]byte, error) { +func (m *Manifest) MarshalJSON() ([]byte, error) { return json.Marshal(m.Manifest) } -func (m *OCIManifest) GetLayers(include func(d ocispec.Descriptor) bool) []ocispec.Descriptor { +// GetLayers returns all the layers in the manifest +func (m *Manifest) GetLayers(include func(d ocispec.Descriptor) bool) []ocispec.Descriptor { var layers []ocispec.Descriptor for _, layer := range m.Layers { if include != nil && include(layer) { diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index c42b7bc2ac..7ccf5a5355 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -65,12 +65,12 @@ func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, destinationDir, doneSaving, encounteredErr, wg) + return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, doneSaving, encounteredErr, wg) } // CopyWithProgress copies the given layers from the remote repository to the given store. func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, - copyOpts oras.CopyOptions, destinationDir string, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) error { + copyOpts oras.CopyOptions, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) error { estimatedBytes := int64(0) shas := []string{} for _, layer := range layers { diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 9893616f16..063c9767cb 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -76,7 +76,8 @@ func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descr } // PublishPackage publishes the package to the remote repository. -func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annotations map[string]string, arch string, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { +func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annotations map[string]string, + desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency // assumes referrers API is not supported since OCI artifact @@ -102,11 +103,7 @@ func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annota return err } - if err := o.UpdateIndex(o.repo.Reference.Reference, publishedDesc); err != nil { - return err - } - - return nil + return o.UpdateIndex(o.repo.Reference.Reference, publishedDesc) } // UpdateIndex updates the index for the given package. diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index c240922f66..42469e3c07 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -44,9 +44,10 @@ func RemoveDuplicateDescriptors(descriptors []ocispec.Descriptor) []ocispec.Desc return list } -func SumLayersSize(layers []ocispec.Descriptor) int64 { +// SumDescsSize returns the size of all the descriptors added together +func SumDescsSize(descs []ocispec.Descriptor) int64 { var sum int64 - for _, layer := range layers { + for _, layer := range descs { sum += layer.Size } return sum diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index e291f043f6..e47146c31a 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. package ocizarf import ( @@ -15,12 +19,13 @@ var ( SkeletonArch = "skeleton" ) +// ZarfOrasRemote is a wrapper around the Oras remote repository with zarf specific functions type ZarfOrasRemote struct { *oci.OrasRemote } -type Modifier func(*oci.OrasRemote) - +// NewZarfOrasRemote returns an oras remote repository client and context for the given url +// with zarf opination embedded func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { modifiers := append(mod, oci.WithMediaType(ZarfConfigMediaType)) remote, err := oci.NewOrasRemote(url, &message.Logger{}, platform, modifiers...) diff --git a/src/pkg/ocizarf/copier.go b/src/pkg/ocizarf/copier.go index 95c2b4056c..239346c9bf 100644 --- a/src/pkg/ocizarf/copier.go +++ b/src/pkg/ocizarf/copier.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. package ocizarf import ( @@ -13,6 +13,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) +// CopyPackage copies a zarf package from one OCI registry to another func CopyPackage(ctx context.Context, src *ZarfOrasRemote, dst *ZarfOrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { @@ -21,14 +22,11 @@ func CopyPackage(ctx context.Context, src *ZarfOrasRemote, dst *ZarfOrasRemote, return err } layers := srcRoot.GetLayers(include) - size := oci.SumLayersSize(srcRoot.Layers) + size := oci.SumDescsSize(srcRoot.Layers) title := fmt.Sprintf("[0/%d] layers copied", len(layers)) progressBar := message.NewProgressBar(size, title) defer progressBar.Finish(err, "Copied %s", src.Repo().Reference) - if err = oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, include, concurrency, progressBar); err != nil { - return err - } - return nil + return oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, include, concurrency, progressBar) } diff --git a/src/pkg/ocizarf/fetch.go b/src/pkg/ocizarf/fetch.go index 65d58c0662..dfc3c81df8 100644 --- a/src/pkg/ocizarf/fetch.go +++ b/src/pkg/ocizarf/fetch.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. package ocizarf import ( diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index 5d95052529..efa1ea2717 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. package ocizarf import ( @@ -65,7 +65,7 @@ func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, lay wg.Add(1) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+o.Repo().Reference.String()) - layerSize := oci.SumLayersSize(layersToPull) + layerSize := oci.SumDescsSize(layersToPull) go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) return o.PullLayers(destinationDir, concurrency, layersToPull, doneSaving, encounteredErr, &wg) diff --git a/src/pkg/ocizarf/push.go b/src/pkg/ocizarf/push.go index c6e724732f..64bdb05ab9 100644 --- a/src/pkg/ocizarf/push.go +++ b/src/pkg/ocizarf/push.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. package ocizarf import ( @@ -17,6 +17,7 @@ import ( "oras.land/oras-go/v2/content/file" ) +// PublishZarfPackage publishes the zarf package to the remote repository. func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int) error { src, err := file.New(paths.Base) if err != nil { @@ -52,7 +53,7 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) annotations := zarfPackageOciAnnotations(&pkg.Metadata) - err = o.PublishPackage(ctx, src, annotations, pkg.Build.Architecture, descs, config.CommonOptions.OCIConcurrency, progressBar) + err = o.PublishPackage(ctx, src, annotations, descs, config.CommonOptions.OCIConcurrency, progressBar) if err != nil { progressBar.Stop() return fmt.Errorf("unable to publish package: %w", err) diff --git a/src/pkg/ocizarf/utils.go b/src/pkg/ocizarf/utils.go index 3652359ed5..5f10b13c18 100644 --- a/src/pkg/ocizarf/utils.go +++ b/src/pkg/ocizarf/utils.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. package ocizarf import ( @@ -5,6 +9,7 @@ import ( "fmt" "strings" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "oras.land/oras-go/v2/registry" @@ -29,7 +34,7 @@ func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata raw = fmt.Sprintf("%s-%s", raw, build.Flavor) } - //o.log("Raw OCI reference from metadata:", raw) + message.Debug("Raw OCI reference from metadata:", raw) ref, err := registry.ParseReference(raw) if err != nil { diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index ef30b3f97d..831a15f410 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -106,9 +106,9 @@ func (ic *ImportChain) fetchOCISkeleton() error { var wg sync.WaitGroup wg.Add(1) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) - layerSize := oci.SumLayersSize([]ocispec.Descriptor{componentDesc}) + layerSize := oci.SumDescsSize([]ocispec.Descriptor{componentDesc}) go utils.RenderProgressBarForLocalDirWrite(cache, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) - if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache, doneSaving, encounteredErr, &wg); err != nil { + if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, doneSaving, encounteredErr, &wg); err != nil { return err } } diff --git a/src/pkg/utils/helpers/transport.go b/src/pkg/utils/helpers/transport.go index 3172877d40..d6e19cc0e1 100644 --- a/src/pkg/utils/helpers/transport.go +++ b/src/pkg/utils/helpers/transport.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic utility functions. +// Package helpers provides generic utility functions. package helpers import ( From 9cb21d78cc5e3c533dc4ed893728f29fcc54ec89 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 1 Feb 2024 15:29:10 +0000 Subject: [PATCH 032/113] moving ctx out of orasremote --- src/pkg/oci/common.go | 14 --------- src/pkg/oci/copier.go | 2 +- src/pkg/oci/fetch.go | 33 ++++++++++---------- src/pkg/oci/pull.go | 20 ++++++------- src/pkg/oci/push.go | 35 +++++++++++----------- src/pkg/ocizarf/common.go | 3 ++ src/pkg/ocizarf/copier.go | 5 ++-- src/pkg/ocizarf/fetch.go | 14 +++++---- src/pkg/ocizarf/pull.go | 25 ++++++++-------- src/pkg/packager/composer/list.go | 3 +- src/pkg/packager/composer/oci.go | 7 +++-- src/pkg/packager/create_stages.go | 2 +- src/pkg/packager/publish.go | 8 ++--- src/pkg/packager/sources/oci.go | 16 +++++----- src/test/e2e/50_oci_publish_deploy_test.go | 2 +- 15 files changed, 93 insertions(+), 96 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 982f3c40ef..c7289c11ce 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -5,7 +5,6 @@ package oci import ( - "context" "fmt" "io" "net/http" @@ -57,7 +56,6 @@ type ProgressWriter interface { type OrasRemote struct { repo *remote.Repository root *Manifest - ctx context.Context Transport *helpers.Transport CopyOpts oras.CopyOptions targetPlatform *ocispec.Platform @@ -69,13 +67,6 @@ type OrasRemote struct { // Modifier is a function that modifies an OrasRemote type Modifier func(*OrasRemote) -// WithContext sets the context for the remote -func WithContext(ctx context.Context) Modifier { - return func(o *OrasRemote) { - o.ctx = ctx - } -} - // WithInsecure sets plainHTTP and insecure tls for the remote func WithInsecure(insecure bool) Modifier { return func(o *OrasRemote) { @@ -154,11 +145,6 @@ func NewOrasRemote(url string, logger Logger, platform ocispec.Platform, mods .. mod(o) } - // if no context is provided, use the default - if o.ctx == nil { - o.ctx = context.TODO() - } - return o, nil } diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 9d2c6b3c60..075736cf20 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -26,7 +26,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, sem := semaphore.NewWeighted(int64(concurrency)) // fetch the source root manifest - srcRoot, err := src.FetchRoot() + srcRoot, err := src.FetchRoot(ctx) if err != nil { return err } diff --git a/src/pkg/oci/fetch.go b/src/pkg/oci/fetch.go index b2c8c84be0..952fde58d8 100644 --- a/src/pkg/oci/fetch.go +++ b/src/pkg/oci/fetch.go @@ -5,6 +5,7 @@ package oci import ( + "context" "encoding/json" "fmt" @@ -16,9 +17,9 @@ import ( ) // ResolveRoot returns the root descriptor for the remote repository -func (o *OrasRemote) ResolveRoot() (ocispec.Descriptor, error) { +func (o *OrasRemote) ResolveRoot(ctx context.Context) (ocispec.Descriptor, error) { // first try to resolve the reference into an OCI descriptor directly - desc, err := o.repo.Resolve(o.ctx, o.repo.Reference.Reference) + desc, err := o.repo.Resolve(ctx, o.repo.Reference.Reference) // if we succeeded and it's not an index, return it // otherwise we will use oras.Resolve which will fetch the index, then resolve the manifest // w/ the target platform @@ -36,22 +37,22 @@ func (o *OrasRemote) ResolveRoot() (ocispec.Descriptor, error) { TargetPlatform: o.targetPlatform, } // if the first attempt failed to resolve, or returned an index, try again with oras.Resolve - return oras.Resolve(o.ctx, o.repo, o.repo.Reference.Reference, resolveOpts) + return oras.Resolve(ctx, o.repo, o.repo.Reference.Reference, resolveOpts) } // FetchRoot fetches the root manifest from the remote repository. -func (o *OrasRemote) FetchRoot() (*Manifest, error) { +func (o *OrasRemote) FetchRoot(ctx context.Context) (*Manifest, error) { if o.root != nil { return o.root, nil } // get the manifest descriptor - descriptor, err := o.ResolveRoot() + descriptor, err := o.ResolveRoot(ctx) if err != nil { return nil, err } // fetch the manifest - root, err := o.FetchManifest(descriptor) + root, err := o.FetchManifest(ctx, descriptor) if err != nil { return nil, err } @@ -60,36 +61,36 @@ func (o *OrasRemote) FetchRoot() (*Manifest, error) { } // FetchManifest fetches the manifest with the given descriptor from the remote repository. -func (o *OrasRemote) FetchManifest(desc ocispec.Descriptor) (manifest *Manifest, err error) { - return FetchUnmarshal[*Manifest](o.FetchLayer, json.Unmarshal, desc) +func (o *OrasRemote) FetchManifest(ctx context.Context, desc ocispec.Descriptor) (manifest *Manifest, err error) { + return FetchUnmarshal[*Manifest](ctx, o.FetchLayer, json.Unmarshal, desc) } // FetchLayer fetches the layer with the given descriptor from the remote repository. -func (o *OrasRemote) FetchLayer(desc ocispec.Descriptor) (bytes []byte, err error) { - return content.FetchAll(o.ctx, o.repo, desc) +func (o *OrasRemote) FetchLayer(ctx context.Context, desc ocispec.Descriptor) (bytes []byte, err error) { + return content.FetchAll(ctx, o.repo, desc) } // FetchJSONFile fetches the given JSON file from the remote repository. -func FetchJSONFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *Manifest, path string) (result T, err error) { +func FetchJSONFile[T any](ctx context.Context, fetcher func(ctx context.Context, desc ocispec.Descriptor) (bytes []byte, err error), manifest *Manifest, path string) (result T, err error) { descriptor := manifest.Locate(path) if IsEmptyDescriptor(descriptor) { return result, fmt.Errorf("unable to find %s in the manifest", path) } - return FetchUnmarshal[T](fetcher, json.Unmarshal, descriptor) + return FetchUnmarshal[T](ctx, fetcher, json.Unmarshal, descriptor) } // FetchYAMLFile fetches the given YAML file from the remote repository. -func FetchYAMLFile[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), manifest *Manifest, path string) (result T, err error) { +func FetchYAMLFile[T any](ctx context.Context, fetcher func(ctx context.Context, desc ocispec.Descriptor) (bytes []byte, err error), manifest *Manifest, path string) (result T, err error) { descriptor := manifest.Locate(path) if IsEmptyDescriptor(descriptor) { return result, fmt.Errorf("unable to find %s in the manifest", path) } - return FetchUnmarshal[T](fetcher, goyaml.Unmarshal, descriptor) + return FetchUnmarshal[T](ctx, fetcher, goyaml.Unmarshal, descriptor) } // FetchUnmarshal fetches the given descriptor from the remote repository and unmarshals it. -func FetchUnmarshal[T any](fetcher func(desc ocispec.Descriptor) (bytes []byte, err error), unmarshaler func(data []byte, v interface{}) error, descriptor ocispec.Descriptor) (result T, err error) { - bytes, err := fetcher(descriptor) +func FetchUnmarshal[T any](ctx context.Context, fetcher func(ctx context.Context, desc ocispec.Descriptor) (bytes []byte, err error), unmarshaler func(data []byte, v interface{}) error, descriptor ocispec.Descriptor) (result T, err error) { + bytes, err := fetcher(ctx, descriptor) if err != nil { return result, err } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 7ccf5a5355..895443771b 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -51,7 +51,7 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi // PullLayers pulls the package from the remote repository and saves it to the given path. // If you don't have nil paramaters for doneSaving, encounteredErr, and wg // you must use the channels in a go routine and call wg.done after they are used -func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, +func (o *OrasRemote) PullLayers(ctx context.Context, destinationDir string, concurrency int, layersToPull []ocispec.Descriptor, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) ([]ocispec.Descriptor, error) { // de-duplicate layers layersToPull = RemoveDuplicateDescriptors(layersToPull) @@ -65,11 +65,11 @@ func (o *OrasRemote) PullLayers(destinationDir string, concurrency int, copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, doneSaving, encounteredErr, wg) + return layersToPull, o.CopyWithProgress(ctx, layersToPull, dst, copyOpts, doneSaving, encounteredErr, wg) } // CopyWithProgress copies the given layers from the remote repository to the given store. -func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, +func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Descriptor, store oras.Target, copyOpts oras.CopyOptions, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) error { estimatedBytes := int64(0) shas := []string{} @@ -91,7 +91,7 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta nodes = []ocispec.Descriptor{} // expand the manifests for _, node := range manifestDescs { - manifest, err := o.FetchManifest(node) + manifest, err := o.FetchManifest(ctx, node) if err != nil { return nil, err } @@ -110,7 +110,7 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta } } - _, err := oras.Copy(o.ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) + _, err := oras.Copy(ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) if err != nil { if encounteredErr != nil { encounteredErr <- 1 @@ -132,8 +132,8 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta // PullLayer pulls a layer from the remote repository and saves it to `destinationDir/annotationTitle`. // ?! Why do we pull a single layer with o.fetchLayer, but multiple layers with oras.copy -func (o *OrasRemote) PullLayer(desc ocispec.Descriptor, destinationDir string) error { - b, err := o.FetchLayer(desc) +func (o *OrasRemote) PullLayer(ctx context.Context, desc ocispec.Descriptor, destinationDir string) error { + b, err := o.FetchLayer(ctx, desc) if err != nil { return err } @@ -144,9 +144,9 @@ func (o *OrasRemote) PullLayer(desc ocispec.Descriptor, destinationDir string) e } // PullFilesAtPaths pulls multiple files from the remote repository and saves them to `destinationDir`. -func (o *OrasRemote) PullFilesAtPaths(paths []string, destinationDir string) ([]ocispec.Descriptor, error) { +func (o *OrasRemote) PullFilesAtPaths(ctx context.Context, paths []string, destinationDir string) ([]ocispec.Descriptor, error) { paths = helpers.Unique(paths) - root, err := o.FetchRoot() + root, err := o.FetchRoot(ctx) if err != nil { return nil, err } @@ -158,7 +158,7 @@ func (o *OrasRemote) PullFilesAtPaths(paths []string, destinationDir string) ([] if o.FileDescriptorExists(desc, destinationDir) { continue } - err = o.PullLayer(desc, destinationDir) + err = o.PullLayer(ctx, desc, destinationDir) if err != nil { return nil, err } diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 063c9767cb..2610c438f9 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -34,13 +34,13 @@ type ConfigPartial struct { } // PushLayer pushes the given layer (bytes) to the remote repository. -func (o *OrasRemote) PushLayer(b []byte, mediaType string) (*ocispec.Descriptor, error) { +func (o *OrasRemote) PushLayer(ctx context.Context, b []byte, mediaType string) (*ocispec.Descriptor, error) { desc := content.NewDescriptorFromBytes(mediaType, b) - return &desc, o.repo.Push(o.ctx, desc, bytes.NewReader(b)) + return &desc, o.repo.Push(ctx, desc, bytes.NewReader(b)) } // pushManifestConfigFromMetadata pushes the manifest config with metadata to the remote repository. -func (o *OrasRemote) pushManifestConfigFromMetadata(annotations map[string]string) (*ocispec.Descriptor, error) { +func (o *OrasRemote) pushManifestConfigFromMetadata(ctx context.Context, annotations map[string]string) (*ocispec.Descriptor, error) { if annotations[ocispec.AnnotationTitle] == "" { return nil, fmt.Errorf("invalid annotations: please include value for %q", ocispec.AnnotationTitle) } @@ -54,21 +54,22 @@ func (o *OrasRemote) pushManifestConfigFromMetadata(annotations map[string]strin return nil, err } // If Media type is not set it will be set to the default - return o.PushLayer(manifestConfigBytes, o.mediaType) + return o.PushLayer(ctx, manifestConfigBytes, o.mediaType) } -func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descriptor, configDesc *ocispec.Descriptor, annotations map[string]string) (ocispec.Descriptor, error) { +func (o *OrasRemote) generatePackManifest(ctx context.Context, src *file.Store, descs []ocispec.Descriptor, + configDesc *ocispec.Descriptor, annotations map[string]string) (ocispec.Descriptor, error) { packOpts := oras.PackManifestOptions{ Layers: descs, ConfigDescriptor: configDesc, ManifestAnnotations: annotations, } - root, err := oras.PackManifest(o.ctx, src, oras.PackManifestVersion1_1_RC4, "", packOpts) + root, err := oras.PackManifest(ctx, src, oras.PackManifestVersion1_1_RC4, "", packOpts) if err != nil { return ocispec.Descriptor{}, err } - if err = src.Tag(o.ctx, root, root.Digest.String()); err != nil { + if err = src.Tag(ctx, root, root.Digest.String()); err != nil { return ocispec.Descriptor{}, err } @@ -87,11 +88,11 @@ func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annota // push the manifest config // since this config is so tiny, and the content is not used again // it is not logged to the progress, but will error if it fails - manifestConfigDesc, err := o.pushManifestConfigFromMetadata(annotations) + manifestConfigDesc, err := o.pushManifestConfigFromMetadata(ctx, annotations) if err != nil { return err } - root, err := o.generatePackManifest(src, desc, manifestConfigDesc, annotations) + root, err := o.generatePackManifest(ctx, src, desc, manifestConfigDesc, annotations) if err != nil { return err } @@ -103,18 +104,18 @@ func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annota return err } - return o.UpdateIndex(o.repo.Reference.Reference, publishedDesc) + return o.UpdateIndex(ctx, o.repo.Reference.Reference, publishedDesc) } // UpdateIndex updates the index for the given package. -func (o *OrasRemote) UpdateIndex(tag string, publishedDesc ocispec.Descriptor) error { +func (o *OrasRemote) UpdateIndex(ctx context.Context, tag string, publishedDesc ocispec.Descriptor) error { var index ocispec.Index o.repo.Reference.Reference = tag // since ref has changed, need to reset root o.root = nil - _, err := o.repo.Resolve(o.ctx, o.repo.Reference.Reference) + _, err := o.repo.Resolve(ctx, o.repo.Reference.Reference) if err != nil { if errors.Is(err, errdef.ErrNotFound) { index = ocispec.Index{ @@ -130,12 +131,12 @@ func (o *OrasRemote) UpdateIndex(tag string, publishedDesc ocispec.Descriptor) e }, }, } - return o.pushIndex(&index, tag) + return o.pushIndex(ctx, &index, tag) } return err } - desc, rc, err := o.repo.FetchReference(o.ctx, tag) + desc, rc, err := o.repo.FetchReference(ctx, tag) if err != nil { return err } @@ -169,14 +170,14 @@ func (o *OrasRemote) UpdateIndex(tag string, publishedDesc ocispec.Descriptor) e }) } - return o.pushIndex(&index, tag) + return o.pushIndex(ctx, &index, tag) } -func (o *OrasRemote) pushIndex(index *ocispec.Index, tag string) error { +func (o *OrasRemote) pushIndex(ctx context.Context, index *ocispec.Index, tag string) error { indexBytes, err := json.Marshal(index) if err != nil { return err } indexDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeImageIndex, indexBytes) - return o.repo.Manifests().PushReference(o.ctx, indexDesc, bytes.NewReader(indexBytes), tag) + return o.repo.Manifests().PushReference(ctx, indexDesc, bytes.NewReader(indexBytes), tag) } diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index e47146c31a..abb14b1f79 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -24,6 +24,9 @@ type ZarfOrasRemote struct { *oci.OrasRemote } +//TODO: take away ctx from ORAS remote +//TODO: Look in ocizarf to see if there's any functions that should be moved outside of oci completely + // NewZarfOrasRemote returns an oras remote repository client and context for the given url // with zarf opination embedded func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { diff --git a/src/pkg/ocizarf/copier.go b/src/pkg/ocizarf/copier.go index 239346c9bf..dda63ae5ee 100644 --- a/src/pkg/ocizarf/copier.go +++ b/src/pkg/ocizarf/copier.go @@ -14,10 +14,9 @@ import ( ) // CopyPackage copies a zarf package from one OCI registry to another -func CopyPackage(ctx context.Context, src *ZarfOrasRemote, dst *ZarfOrasRemote, - include func(d ocispec.Descriptor) bool, concurrency int) error { +func CopyPackage(ctx context.Context, src *ZarfOrasRemote, dst *ZarfOrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { - srcRoot, err := src.FetchRoot() + srcRoot, err := src.FetchRoot(ctx) if err != nil { return err } diff --git a/src/pkg/ocizarf/fetch.go b/src/pkg/ocizarf/fetch.go index dfc3c81df8..e80478a268 100644 --- a/src/pkg/ocizarf/fetch.go +++ b/src/pkg/ocizarf/fetch.go @@ -5,6 +5,8 @@ package ocizarf import ( + "context" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/types" @@ -12,19 +14,19 @@ import ( ) // FetchZarfYAML fetches the zarf.yaml file from the remote repository. -func (o *ZarfOrasRemote) FetchZarfYAML() (pkg types.ZarfPackage, err error) { - manifest, err := o.FetchRoot() +func (o *ZarfOrasRemote) FetchZarfYAML(ctx context.Context) (pkg types.ZarfPackage, err error) { + manifest, err := o.FetchRoot(ctx) if err != nil { return pkg, err } - return oci.FetchYAMLFile[types.ZarfPackage](o.FetchLayer, manifest, layout.ZarfYAML) + return oci.FetchYAMLFile[types.ZarfPackage](ctx, o.FetchLayer, manifest, layout.ZarfYAML) } // FetchImagesIndex fetches the images/index.json file from the remote repository. -func (o *ZarfOrasRemote) FetchImagesIndex() (index *ocispec.Index, err error) { - manifest, err := o.FetchRoot() +func (o *ZarfOrasRemote) FetchImagesIndex(ctx context.Context) (index *ocispec.Index, err error) { + manifest, err := o.FetchRoot(ctx) if err != nil { return index, err } - return oci.FetchJSONFile[*ocispec.Index](o.FetchLayer, manifest, ZarfPackageIndexPath) + return oci.FetchJSONFile[*ocispec.Index](ctx, o.FetchLayer, manifest, ZarfPackageIndexPath) } diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index efa1ea2717..630112ee6a 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -5,6 +5,7 @@ package ocizarf import ( + "context" "fmt" "path/filepath" "slices" @@ -39,11 +40,11 @@ var ( // - zarf.yaml // - checksums.txt // - zarf.yaml.sig -func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { +func (o *ZarfOrasRemote) PullPackage(ctx context.Context, destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { isPartialPull := len(layersToPull) > 0 message.Debugf("Pulling %s", o.Repo().Reference) - manifest, err := o.FetchRoot() + manifest, err := o.FetchRoot(ctx) if err != nil { return nil, err } @@ -68,20 +69,20 @@ func (o *ZarfOrasRemote) PullPackage(destinationDir string, concurrency int, lay layerSize := oci.SumDescsSize(layersToPull) go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) - return o.PullLayers(destinationDir, concurrency, layersToPull, doneSaving, encounteredErr, &wg) + return o.PullLayers(ctx, destinationDir, concurrency, layersToPull, doneSaving, encounteredErr, &wg) } // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. // It also retrieves the descriptors for all image layers that are required by the components. // // It also respects the `required` flag on components, and will retrieve all necessary layers for required components. -func LayersFromRequestedComponents(o *ZarfOrasRemote, requestedComponents []string) (layers []ocispec.Descriptor, err error) { - root, err := o.FetchRoot() +func (o *ZarfOrasRemote) LayersFromRequestedComponents(ctx context.Context, requestedComponents []string) (layers []ocispec.Descriptor, err error) { + root, err := o.FetchRoot(ctx) if err != nil { return nil, err } - pkg, err := o.FetchZarfYAML() + pkg, err := o.FetchZarfYAML(ctx) if err != nil { return nil, err } @@ -114,7 +115,7 @@ func LayersFromRequestedComponents(o *ZarfOrasRemote, requestedComponents []stri if len(images) > 0 { // Add the image index and the oci-layout layers layers = append(layers, root.Locate(ZarfPackageIndexPath), root.Locate(ZarfPackageLayoutPath)) - index, err := o.FetchImagesIndex() + index, err := o.FetchImagesIndex(ctx) if err != nil { return nil, err } @@ -135,7 +136,7 @@ func LayersFromRequestedComponents(o *ZarfOrasRemote, requestedComponents []stri // even though these are technically image manifests, we store them as Zarf blobs manifestDescriptor.MediaType = ZarfLayerMediaTypeBlob - manifest, err := o.FetchManifest(manifestDescriptor) + manifest, err := o.FetchManifest(ctx, manifestDescriptor) if err != nil { return nil, err } @@ -154,11 +155,11 @@ func LayersFromRequestedComponents(o *ZarfOrasRemote, requestedComponents []stri } // PullPackageMetadata pulls the package metadata from the remote repository and saves it to `destinationDir`. -func (o *ZarfOrasRemote) PullPackageMetadata(destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullFilesAtPaths(PackageAlwaysPull, destinationDir) +func (o *ZarfOrasRemote) PullPackageMetadata(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { + return o.PullFilesAtPaths(ctx, PackageAlwaysPull, destinationDir) } // PullPackageSBOM pulls the package's sboms.tar from the remote repository and saves it to `destinationDir`. -func (o *ZarfOrasRemote) PullPackageSBOM(destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullFilesAtPaths([]string{layout.SBOMTar}, destinationDir) +func (o *ZarfOrasRemote) PullPackageSBOM(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { + return o.PullFilesAtPaths(ctx, []string{layout.SBOMTar}, destinationDir) } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index caa26f821f..f006959685 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -5,6 +5,7 @@ package composer import ( + "context" "fmt" "path/filepath" "strings" @@ -181,7 +182,7 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar if err != nil { return ic, err } - pkg, err = remote.FetchZarfYAML() + pkg, err = remote.FetchZarfYAML(context.TODO()) if err != nil { return ic, err } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 831a15f410..5dccd746dc 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -34,7 +34,7 @@ func (ic *ImportChain) getRemote(url string) (*ocizarf.ZarfOrasRemote, error) { if err != nil { return nil, err } - _, err = ic.remote.ResolveRoot() + _, err = ic.remote.ResolveRoot(context.TODO()) if err != nil { return nil, fmt.Errorf("published skeleton package for %q does not exist: %w", url, err) } @@ -57,7 +57,8 @@ func (ic *ImportChain) fetchOCISkeleton() error { return err } - manifest, err := remote.FetchRoot() + ctx := context.TODO() + manifest, err := remote.FetchRoot(ctx) if err != nil { return err } @@ -108,7 +109,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) layerSize := oci.SumDescsSize([]ocispec.Descriptor{componentDesc}) go utils.RenderProgressBarForLocalDirWrite(cache, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) - if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, doneSaving, encounteredErr, &wg); err != nil { + if err := remote.CopyWithProgress(ctx, []ocispec.Descriptor{componentDesc}, store, copyOpts, doneSaving, encounteredErr, &wg); err != nil { return err } } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 709d5129e6..0390f80a4b 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -659,7 +659,7 @@ func (p *Packager) loadDifferentialData() error { if err != nil { return err } - pkg, err := remote.FetchZarfYAML() + pkg, err := remote.FetchZarfYAML(context.TODO()) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index addbcfbd27..059df45f96 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -43,12 +43,12 @@ func (p *Packager) Publish() (err error) { return err } - srcRoot, err := srcRemote.ResolveRoot() + srcRoot, err := srcRemote.ResolveRoot(ctx) if err != nil { return err } - pkg, err := srcRemote.FetchZarfYAML() + pkg, err := srcRemote.FetchZarfYAML(ctx) if err != nil { return err } @@ -62,7 +62,7 @@ func (p *Packager) Publish() (err error) { return err } - srcManifest, err := srcRemote.FetchRoot() + srcManifest, err := srcRemote.FetchRoot(ctx) if err != nil { return err } @@ -77,7 +77,7 @@ func (p *Packager) Publish() (err error) { } tag := srcRemote.Repo().Reference.Reference - if err := dstRemote.UpdateIndex(tag, expected); err != nil { + if err := dstRemote.UpdateIndex(ctx, tag, expected); err != nil { return err } message.Infof("Published %s to %s", srcRemote.Repo().Reference, dstRemote.Repo().Reference) diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 8645b38554..70c6d15e95 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -5,6 +5,7 @@ package sources import ( + "context" "errors" "fmt" "os" @@ -35,6 +36,7 @@ type OCISource struct { // LoadPackage loads a package from an OCI registry. func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (err error) { + ctx := context.TODO() var pkg types.ZarfPackage layersToPull := []ocispec.Descriptor{} @@ -45,14 +47,14 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er // pull only needed layers if --confirm is set if config.CommonOptions.Confirm { - layersToPull, err = ocizarf.LayersFromRequestedComponents(s.ZarfOrasRemote, optionalComponents) + layersToPull, err = s.LayersFromRequestedComponents(ctx, optionalComponents) if err != nil { return fmt.Errorf("unable to get published component image layers: %s", err.Error()) } } isPartial := true - root, err := s.FetchRoot() + root, err := s.FetchRoot(ctx) if err != nil { return err } @@ -60,7 +62,7 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er isPartial = false } - layersFetched, err := s.PullPackage(dst.Base, config.CommonOptions.OCIConcurrency, layersToPull...) + layersFetched, err := s.PullPackage(ctx, dst.Base, config.CommonOptions.OCIConcurrency, layersToPull...) if err != nil { return fmt.Errorf("unable to pull the package: %w", err) } @@ -121,8 +123,8 @@ func (s *OCISource) LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, if wantSBOM { toPull = append(toPull, layout.SBOMTar) } - - layersFetched, err := s.PullFilesAtPaths(toPull, dst.Base) + ctx := context.TODO() + layersFetched, err := s.PullFilesAtPaths(ctx, toPull, dst.Base) if err != nil { return err } @@ -174,8 +176,8 @@ func (s *OCISource) Collect(dir string) (string, error) { return "", err } defer os.RemoveAll(tmp) - - fetched, err := s.PullPackage(tmp, config.CommonOptions.OCIConcurrency) + ctx := context.TODO() + fetched, err := s.PullPackage(ctx, tmp, config.CommonOptions.OCIConcurrency) if err != nil { return "", err } diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 3dfa2d4115..53daedf35c 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -153,7 +153,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { err = ocizarf.CopyPackage(ctx, src, dst, nil, 5) suite.NoError(err) - srcRoot, err := src.FetchRoot() + srcRoot, err := src.FetchRoot(ctx) suite.NoError(err) for _, layer := range srcRoot.Layers { From 644a7fa65e1d6c30a039945832d69c52f497b55f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 1 Feb 2024 15:36:08 +0000 Subject: [PATCH 033/113] remove comments --- src/pkg/ocizarf/common.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index abb14b1f79..e47146c31a 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -24,9 +24,6 @@ type ZarfOrasRemote struct { *oci.OrasRemote } -//TODO: take away ctx from ORAS remote -//TODO: Look in ocizarf to see if there's any functions that should be moved outside of oci completely - // NewZarfOrasRemote returns an oras remote repository client and context for the given url // with zarf opination embedded func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { From dff24c8845f80606075f627932cd107d80abbc1a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 1 Feb 2024 18:35:17 +0000 Subject: [PATCH 034/113] Abstracting zarf logic into ZarfOrasRemote --- src/cmd/initialize.go | 2 +- src/cmd/tools/zarf.go | 2 +- src/pkg/ocizarf/common.go | 3 ++- src/pkg/packager/composer/oci.go | 2 +- src/pkg/packager/create_stages.go | 4 ++-- src/pkg/packager/publish.go | 4 ++-- src/pkg/packager/sources/new.go | 2 +- src/pkg/utils/bytes.go | 1 + src/test/e2e/50_oci_publish_deploy_test.go | 6 ++++-- 9 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 9ecab43f57..1a135da886 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -139,7 +139,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { // If the user wants to download the init-package, download it if confirmDownload { - remote, err := ocizarf.NewZarfOrasRemote(url, oci.PlatformForArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(url, oci.PlatformForArch(config.GetArch())) if err != nil { return "", err } diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index d7910a1a25..96280702d0 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -184,7 +184,7 @@ var downloadInitCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { url := ocizarf.GetInitPackageURL(config.CLIVersion) - remote, err := ocizarf.NewZarfOrasRemote(url, oci.PlatformForArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(url, oci.PlatformForArch(config.GetArch())) if err != nil { message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } diff --git a/src/pkg/ocizarf/common.go b/src/pkg/ocizarf/common.go index e47146c31a..adb84418ba 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/ocizarf/common.go @@ -5,6 +5,7 @@ package ocizarf import ( + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -27,7 +28,7 @@ type ZarfOrasRemote struct { // NewZarfOrasRemote returns an oras remote repository client and context for the given url // with zarf opination embedded func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { - modifiers := append(mod, oci.WithMediaType(ZarfConfigMediaType)) + modifiers := append(mod, oci.WithMediaType(ZarfConfigMediaType), oci.WithInsecure(config.CommonOptions.Insecure)) remote, err := oci.NewOrasRemote(url, &message.Logger{}, platform, modifiers...) if err != nil { return nil, err diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 5dccd746dc..dc0e5f4b45 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -30,7 +30,7 @@ func (ic *ImportChain) getRemote(url string) (*ocizarf.ZarfOrasRemote, error) { return ic.remote, nil } var err error - ic.remote, err = ocizarf.NewZarfOrasRemote(url, ocizarf.PlatformForSkeleton(), oci.WithInsecure(config.CommonOptions.Insecure)) + ic.remote, err = ocizarf.NewZarfOrasRemote(url, ocizarf.PlatformForSkeleton()) if err != nil { return nil, err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 0390f80a4b..26e5922d64 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -252,7 +252,7 @@ func (p *Packager) output() error { if err != nil { return err } - remote, err := ocizarf.NewZarfOrasRemote(ref, oci.PlatformForArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(ref, oci.PlatformForArch(config.GetArch())) if err != nil { return err } @@ -655,7 +655,7 @@ func (p *Packager) loadDifferentialData() error { // Load the package spec of the package we're using as a 'reference' for the differential build if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := ocizarf.NewZarfOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch()), oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch())) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 059df45f96..b8d4778ec5 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -38,7 +38,7 @@ func (p *Packager) Publish() (err error) { p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName arch := config.GetArch() - dstRemote, err := ocizarf.NewZarfOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch), oci.WithInsecure(config.CommonOptions.Insecure)) + dstRemote, err := ocizarf.NewZarfOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch)) if err != nil { return err } @@ -118,7 +118,7 @@ func (p *Packager) Publish() (err error) { } else { platform = oci.PlatformForArch(config.GetArch()) } - remote, err := ocizarf.NewZarfOrasRemote(ref, platform, oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(ref, platform) if err != nil { return err } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 7cd9be452d..01abc0c809 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -68,7 +68,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } arch := config.GetArch() - remote, err := ocizarf.NewZarfOrasRemote(pkgSrc, oci.PlatformForArch(arch), oci.WithInsecure(config.CommonOptions.Insecure)) + remote, err := ocizarf.NewZarfOrasRemote(pkgSrc, oci.PlatformForArch(arch)) if err != nil { return nil, err } diff --git a/src/pkg/utils/bytes.go b/src/pkg/utils/bytes.go index 464e11e061..80bc0816c5 100644 --- a/src/pkg/utils/bytes.go +++ b/src/pkg/utils/bytes.go @@ -65,6 +65,7 @@ func RenderProgressBarForLocalDirWrite(filepath string, expectedTotal int64, wg progressBar := message.NewProgressBar(expectedTotal, title) for { + // Could play around with only one select { case <-completeChan: // Send success message diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 53daedf35c..f82ab5e333 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/stretchr/testify/require" @@ -129,10 +130,11 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - src, err := ocizarf.NewZarfOrasRemote(ref, oci.PlatformForArch(e2e.Arch), oci.WithInsecure(true)) + config.CommonOptions.Insecure = true + src, err := ocizarf.NewZarfOrasRemote(ref, oci.PlatformForArch(e2e.Arch)) suite.NoError(err) - dst, err := ocizarf.NewZarfOrasRemote(dstRef, oci.PlatformForArch(e2e.Arch), oci.WithInsecure(true)) + dst, err := ocizarf.NewZarfOrasRemote(dstRef, oci.PlatformForArch(e2e.Arch)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) From 744e5bf2009b5fe6c6de9298232f14ddd4daca8b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 1 Feb 2024 19:32:13 +0000 Subject: [PATCH 035/113] simplyfing concurrency...hopefully --- src/internal/packager/images/pull.go | 20 ++++++++------------ src/pkg/oci/pull.go | 19 ++++++++----------- src/pkg/ocizarf/pull.go | 10 +++------- src/pkg/packager/composer/oci.go | 10 +++------- src/pkg/utils/bytes.go | 26 ++++++++++++-------------- 5 files changed, 34 insertions(+), 51 deletions(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 4bb2a8f0c4..01913a0040 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "strings" - "sync" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" @@ -183,12 +182,9 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { spinner.Success() // Create a thread to update a progress bar as we save the image files to disk - doneSaving := make(chan int) - errorSaving := make(chan int) - var progressBarWaitGroup sync.WaitGroup - progressBarWaitGroup.Add(1) + doneSaving := make(chan error) updateText := fmt.Sprintf("Pulling %d images", imageCount) - go utils.RenderProgressBarForLocalDirWrite(i.ImagesPath, totalBytes, &progressBarWaitGroup, doneSaving, errorSaving, updateText, updateText) + go utils.RenderProgressBarForLocalDirWrite(i.ImagesPath, totalBytes, doneSaving, updateText, updateText) // Spawn a goroutine for each layer to write it to disk using crane @@ -323,8 +319,8 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { onLayerWritingError := func(err error) error { // Send a signal to the progress bar that we're done and wait for the thread to finish - errorSaving <- 1 - progressBarWaitGroup.Wait() + doneSaving <- err + <-doneSaving message.WarnErr(err, "Failed to write image layers, trying again up to 3 times...") if strings.HasPrefix(err.Error(), "expected blob size") { message.Warnf("Potential image cache corruption: %s - try clearing cache with \"zarf tools clear-cache\"", err.Error()) @@ -387,8 +383,8 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { onImageSavingError := func(err error) error { // Send a signal to the progress bar that we're done and wait for the thread to finish - errorSaving <- 1 - progressBarWaitGroup.Wait() + doneSaving <- err + <-doneSaving message.WarnErr(err, "Failed to write image config or manifest, trying again up to 3 times...") return err } @@ -423,8 +419,8 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { } // Send a signal to the progress bar that we're done and wait for the thread to finish - doneSaving <- 1 - progressBarWaitGroup.Wait() + doneSaving <- nil + <-doneSaving return imgInfoList, nil } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 895443771b..6d485fe129 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -8,7 +8,6 @@ import ( "context" "os" "path/filepath" - "sync" "slices" @@ -52,7 +51,7 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi // If you don't have nil paramaters for doneSaving, encounteredErr, and wg // you must use the channels in a go routine and call wg.done after they are used func (o *OrasRemote) PullLayers(ctx context.Context, destinationDir string, concurrency int, - layersToPull []ocispec.Descriptor, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) ([]ocispec.Descriptor, error) { + layersToPull []ocispec.Descriptor, doneSaving chan error) ([]ocispec.Descriptor, error) { // de-duplicate layers layersToPull = RemoveDuplicateDescriptors(layersToPull) @@ -65,12 +64,12 @@ func (o *OrasRemote) PullLayers(ctx context.Context, destinationDir string, conc copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - return layersToPull, o.CopyWithProgress(ctx, layersToPull, dst, copyOpts, doneSaving, encounteredErr, wg) + return layersToPull, o.CopyWithProgress(ctx, layersToPull, dst, copyOpts, doneSaving) } // CopyWithProgress copies the given layers from the remote repository to the given store. func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Descriptor, store oras.Target, - copyOpts oras.CopyOptions, doneSaving chan int, encounteredErr chan int, wg *sync.WaitGroup) error { + copyOpts oras.CopyOptions, doneSaving chan error) error { estimatedBytes := int64(0) shas := []string{} for _, layer := range layers { @@ -112,19 +111,17 @@ func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Desc _, err := oras.Copy(ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) if err != nil { - if encounteredErr != nil { - encounteredErr <- 1 + if doneSaving != nil { + doneSaving <- err + <-doneSaving } return err } // Send a signal to the progress bar that we're done and wait for it to finish if doneSaving != nil { - doneSaving <- 1 - } - - if wg != nil { - wg.Wait() + doneSaving <- nil + <-doneSaving } return nil diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/ocizarf/pull.go index 630112ee6a..dcb000ec87 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/ocizarf/pull.go @@ -9,7 +9,6 @@ import ( "fmt" "path/filepath" "slices" - "sync" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -60,16 +59,13 @@ func (o *ZarfOrasRemote) PullPackage(ctx context.Context, destinationDir string, layersToPull = append(layersToPull, manifest.Config) // Create a thread to update a progress bar as we save the package to disk - doneSaving := make(chan int) - encounteredErr := make(chan int) - var wg sync.WaitGroup - wg.Add(1) + doneSaving := make(chan error) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+o.Repo().Reference.String()) layerSize := oci.SumDescsSize(layersToPull) - go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) + go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, doneSaving, "Pulling", successText) - return o.PullLayers(ctx, destinationDir, concurrency, layersToPull, doneSaving, encounteredErr, &wg) + return o.PullLayers(ctx, destinationDir, concurrency, layersToPull, doneSaving) } // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index dc0e5f4b45..cfaa43f7bd 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -10,7 +10,6 @@ import ( "fmt" "os" "path/filepath" - "sync" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" @@ -102,14 +101,11 @@ func (ic *ImportChain) fetchOCISkeleton() error { copyOpts := remote.CopyOpts // TODO (@WSTARR): This overrides the FindSuccessors function to no longer filter nodes when pulling which is necessary when caching - once we implement caching more thoroughly we will need to reevaluate this. copyOpts.FindSuccessors = content.Successors - doneSaving := make(chan int) - encounteredErr := make(chan int) - var wg sync.WaitGroup - wg.Add(1) + doneSaving := make(chan error) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) layerSize := oci.SumDescsSize([]ocispec.Descriptor{componentDesc}) - go utils.RenderProgressBarForLocalDirWrite(cache, layerSize, &wg, doneSaving, encounteredErr, "Pulling", successText) - if err := remote.CopyWithProgress(ctx, []ocispec.Descriptor{componentDesc}, store, copyOpts, doneSaving, encounteredErr, &wg); err != nil { + go utils.RenderProgressBarForLocalDirWrite(cache, layerSize, doneSaving, "Pulling", successText) + if err := remote.CopyWithProgress(ctx, []ocispec.Descriptor{componentDesc}, store, copyOpts, doneSaving); err != nil { return err } } diff --git a/src/pkg/utils/bytes.go b/src/pkg/utils/bytes.go index 80bc0816c5..8d4d269fcd 100644 --- a/src/pkg/utils/bytes.go +++ b/src/pkg/utils/bytes.go @@ -9,7 +9,6 @@ import ( "fmt" "math" "strconv" - "sync" "time" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -58,26 +57,25 @@ func ByteFormat(inputNum float64, precision int) string { // RenderProgressBarForLocalDirWrite creates a progress bar that continuously tracks the progress of writing files to a local directory and all of its subdirectories. // NOTE: This function runs infinitely until either completeChan or errChan is triggered, this function should be run in a goroutine while a different thread/process is writing to the directory. -func RenderProgressBarForLocalDirWrite(filepath string, expectedTotal int64, wg *sync.WaitGroup, completeChan chan int, errChan chan int, updateText string, successText string) { +func RenderProgressBarForLocalDirWrite(filepath string, expectedTotal int64, completeChan chan error, updateText string, successText string) { // Create a progress bar title := fmt.Sprintf("%s (%s of %s)", updateText, ByteFormat(float64(0), 2), ByteFormat(float64(expectedTotal), 2)) progressBar := message.NewProgressBar(expectedTotal, title) for { - // Could play around with only one select { - case <-completeChan: - // Send success message - progressBar.Successf("%s (%s)", successText, ByteFormat(float64(expectedTotal), 2)) - wg.Done() - return - - case <-errChan: - progressBar.Stop() - wg.Done() - return - + case err := <-completeChan: + if err == nil { + // Send success message + progressBar.Successf("%s (%s)", successText, ByteFormat(float64(expectedTotal), 2)) + completeChan <- nil + return + } else { + progressBar.Stop() + completeChan <- nil + return + } default: // Read the directory size currentBytes, dirErr := GetDirSize(filepath) From 73e4a53e157650ebc16eba28dce45e7c46b92bb2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 1 Feb 2024 19:51:37 +0000 Subject: [PATCH 036/113] renaming PlatformForArch --- src/cmd/initialize.go | 2 +- src/cmd/tools/zarf.go | 2 +- src/pkg/oci/common.go | 6 ++++-- src/pkg/oci/pull.go | 7 ++++--- src/pkg/packager/create_stages.go | 4 ++-- src/pkg/packager/publish.go | 4 ++-- src/pkg/packager/sources/new.go | 2 +- src/test/e2e/50_oci_publish_deploy_test.go | 4 ++-- 8 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 1a135da886..5308a5df8a 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -139,7 +139,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { // If the user wants to download the init-package, download it if confirmDownload { - remote, err := ocizarf.NewZarfOrasRemote(url, oci.PlatformForArch(config.GetArch())) + remote, err := ocizarf.NewZarfOrasRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { return "", err } diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index 96280702d0..b519a658b0 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -184,7 +184,7 @@ var downloadInitCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { url := ocizarf.GetInitPackageURL(config.CLIVersion) - remote, err := ocizarf.NewZarfOrasRemote(url, oci.PlatformForArch(config.GetArch())) + remote, err := ocizarf.NewZarfOrasRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index c7289c11ce..cdeb32851f 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -105,8 +105,10 @@ func WithMediaType(mediaType string) Modifier { } } -// PlatformForArch sets the target architecture for the remote -func PlatformForArch(arch string) ocispec.Platform { +// MultiOSPlatformForArch sets the target architecture for the remote +// ?! Do we want to expose this function or put it in the zarf specific land ? +// Same question for the MultiOS variable +func MultiOSPlatformForArch(arch string) ocispec.Platform { return ocispec.Platform{ OS: MultiOS, Architecture: arch, diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 6d485fe129..8fe7450f0f 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -48,8 +48,8 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi } // PullLayers pulls the package from the remote repository and saves it to the given path. -// If you don't have nil paramaters for doneSaving, encounteredErr, and wg -// you must use the channels in a go routine and call wg.done after they are used +// If you don't set doneSaving to nil you must have a goroutine running that acts on an input +// to doneSaving and provides an input to doneSaving after it's done func (o *OrasRemote) PullLayers(ctx context.Context, destinationDir string, concurrency int, layersToPull []ocispec.Descriptor, doneSaving chan error) ([]ocispec.Descriptor, error) { // de-duplicate layers @@ -68,6 +68,8 @@ func (o *OrasRemote) PullLayers(ctx context.Context, destinationDir string, conc } // CopyWithProgress copies the given layers from the remote repository to the given store. +// If you don't set doneSaving to nil you must have a goroutine running that acts on an input +// to doneSaving and provides an input to doneSaving after it's done func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Descriptor, store oras.Target, copyOpts oras.CopyOptions, doneSaving chan error) error { estimatedBytes := int64(0) @@ -128,7 +130,6 @@ func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Desc } // PullLayer pulls a layer from the remote repository and saves it to `destinationDir/annotationTitle`. -// ?! Why do we pull a single layer with o.fetchLayer, but multiple layers with oras.copy func (o *OrasRemote) PullLayer(ctx context.Context, desc ocispec.Descriptor, destinationDir string) error { b, err := o.FetchLayer(ctx, desc) if err != nil { diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 26e5922d64..6ab8e13e0b 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -252,7 +252,7 @@ func (p *Packager) output() error { if err != nil { return err } - remote, err := ocizarf.NewZarfOrasRemote(ref, oci.PlatformForArch(config.GetArch())) + remote, err := ocizarf.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { return err } @@ -655,7 +655,7 @@ func (p *Packager) loadDifferentialData() error { // Load the package spec of the package we're using as a 'reference' for the differential build if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := ocizarf.NewZarfOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch())) + remote, err := ocizarf.NewZarfOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index b8d4778ec5..882530677f 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -38,7 +38,7 @@ func (p *Packager) Publish() (err error) { p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName arch := config.GetArch() - dstRemote, err := ocizarf.NewZarfOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch)) + dstRemote, err := ocizarf.NewZarfOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.MultiOSPlatformForArch(arch)) if err != nil { return err } @@ -116,7 +116,7 @@ func (p *Packager) Publish() (err error) { if p.cfg.CreateOpts.IsSkeleton { platform = ocizarf.PlatformForSkeleton() } else { - platform = oci.PlatformForArch(config.GetArch()) + platform = oci.MultiOSPlatformForArch(config.GetArch()) } remote, err := ocizarf.NewZarfOrasRemote(ref, platform) if err != nil { diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 01abc0c809..3726feb695 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -68,7 +68,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } arch := config.GetArch() - remote, err := ocizarf.NewZarfOrasRemote(pkgSrc, oci.PlatformForArch(arch)) + remote, err := ocizarf.NewZarfOrasRemote(pkgSrc, oci.MultiOSPlatformForArch(arch)) if err != nil { return nil, err } diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index f82ab5e333..88ae91295c 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -131,10 +131,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { defer e2e.TeardownRegistry(t, dstRegistryPort) config.CommonOptions.Insecure = true - src, err := ocizarf.NewZarfOrasRemote(ref, oci.PlatformForArch(e2e.Arch)) + src, err := ocizarf.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(e2e.Arch)) suite.NoError(err) - dst, err := ocizarf.NewZarfOrasRemote(dstRef, oci.PlatformForArch(e2e.Arch)) + dst, err := ocizarf.NewZarfOrasRemote(dstRef, oci.MultiOSPlatformForArch(e2e.Arch)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) From 8e4b93cea3e8fa326068486d03c97c6c837b0481 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 1 Feb 2024 19:55:18 +0000 Subject: [PATCH 037/113] ZOCI --- src/cmd/initialize.go | 6 +++--- src/cmd/tools/zarf.go | 6 +++--- src/pkg/packager/composer/list.go | 4 ++-- src/pkg/packager/composer/oci.go | 6 +++--- src/pkg/packager/create_stages.go | 8 ++++---- src/pkg/packager/publish.go | 12 ++++++------ src/pkg/packager/sources/new.go | 4 ++-- src/pkg/packager/sources/oci.go | 8 ++++---- src/pkg/packager/sources/tarball.go | 4 ++-- src/pkg/{ocizarf => zoci}/common.go | 4 ++-- src/pkg/{ocizarf => zoci}/copier.go | 4 ++-- src/pkg/{ocizarf => zoci}/fetch.go | 4 ++-- src/pkg/{ocizarf => zoci}/pull.go | 4 ++-- src/pkg/{ocizarf => zoci}/push.go | 4 ++-- src/pkg/{ocizarf => zoci}/utils.go | 4 ++-- src/test/e2e/50_oci_publish_deploy_test.go | 8 ++++---- 16 files changed, 45 insertions(+), 45 deletions(-) rename src/pkg/{ocizarf => zoci}/common.go (93%) rename src/pkg/{ocizarf => zoci}/copier.go (88%) rename src/pkg/{ocizarf => zoci}/fetch.go (89%) rename src/pkg/{ocizarf => zoci}/pull.go (98%) rename src/pkg/{ocizarf => zoci}/push.go (95%) rename src/pkg/{ocizarf => zoci}/utils.go (92%) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 5308a5df8a..81bb53751b 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -18,11 +18,11 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/spf13/cobra" ) @@ -120,7 +120,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { } var confirmDownload bool - url := ocizarf.GetInitPackageURL(config.CLIVersion) + url := zoci.GetInitPackageURL(config.CLIVersion) // Give the user the choice to download the init-package and note that this does require an internet connection message.Question(fmt.Sprintf(lang.CmdInitPullAsk, url)) @@ -139,7 +139,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { // If the user wants to download the init-package, download it if confirmDownload { - remote, err := ocizarf.NewZarfOrasRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewZarfOrasRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { return "", err } diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index b519a658b0..4e81f01d4f 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -19,9 +19,9 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/pki" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" "github.com/sigstore/cosign/v2/pkg/cosign" "github.com/spf13/cobra" @@ -182,9 +182,9 @@ var downloadInitCmd = &cobra.Command{ Use: "download-init", Short: lang.CmdToolsDownloadInitShort, Run: func(cmd *cobra.Command, args []string) { - url := ocizarf.GetInitPackageURL(config.CLIVersion) + url := zoci.GetInitPackageURL(config.CLIVersion) - remote, err := ocizarf.NewZarfOrasRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewZarfOrasRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index f006959685..07297ea306 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -12,10 +12,10 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" ) @@ -81,7 +81,7 @@ type ImportChain struct { head *Node tail *Node - remote *ocizarf.ZarfOrasRemote + remote *zoci.ZarfOrasRemote } // Head returns the first node in the import chain diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index cfaa43f7bd..c61e45cdc8 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -15,21 +15,21 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/content" ocistore "oras.land/oras-go/v2/content/oci" ) -func (ic *ImportChain) getRemote(url string) (*ocizarf.ZarfOrasRemote, error) { +func (ic *ImportChain) getRemote(url string) (*zoci.ZarfOrasRemote, error) { if ic.remote != nil { return ic.remote, nil } var err error - ic.remote, err = ocizarf.NewZarfOrasRemote(url, ocizarf.PlatformForSkeleton()) + ic.remote, err = zoci.NewZarfOrasRemote(url, zoci.PlatformForSkeleton()) if err != nil { return nil, err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 6ab8e13e0b..22b0536a5f 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -25,10 +25,10 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" "github.com/go-git/go-git/v5/plumbing" "github.com/mholt/archiver/v3" @@ -248,11 +248,11 @@ func (p *Packager) output() error { // Create a remote ref + client for the package (if output is OCI) // then publish the package to the remote. if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { - ref, err := ocizarf.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) + ref, err := zoci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) if err != nil { return err } - remote, err := ocizarf.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { return err } @@ -655,7 +655,7 @@ func (p *Packager) loadDifferentialData() error { // Load the package spec of the package we're using as a 'reference' for the differential build if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := ocizarf.NewZarfOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewZarfOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 882530677f..dadf3299cf 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -14,10 +14,10 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/content" @@ -38,7 +38,7 @@ func (p *Packager) Publish() (err error) { p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName arch := config.GetArch() - dstRemote, err := ocizarf.NewZarfOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.MultiOSPlatformForArch(arch)) + dstRemote, err := zoci.NewZarfOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.MultiOSPlatformForArch(arch)) if err != nil { return err } @@ -58,7 +58,7 @@ func (p *Packager) Publish() (err error) { return fmt.Errorf("architecture mismatch (specified: %q, found %q)", arch, pkg.Build.Architecture) } - if err := ocizarf.CopyPackage(ctx, srcRemote, dstRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { + if err := zoci.CopyPackage(ctx, srcRemote, dstRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { return err } @@ -108,17 +108,17 @@ func (p *Packager) Publish() (err error) { } // Get a reference to the registry for this package - ref, err := ocizarf.ReferenceFromMetadata(p.cfg.PublishOpts.PackageDestination, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) + ref, err := zoci.ReferenceFromMetadata(p.cfg.PublishOpts.PackageDestination, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) if err != nil { return err } var platform ocispec.Platform if p.cfg.CreateOpts.IsSkeleton { - platform = ocizarf.PlatformForSkeleton() + platform = zoci.PlatformForSkeleton() } else { platform = oci.MultiOSPlatformForArch(config.GetArch()) } - remote, err := ocizarf.NewZarfOrasRemote(ref, platform) + remote, err := zoci.NewZarfOrasRemote(ref, platform) if err != nil { return err } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 3726feb695..bf8a248203 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -13,8 +13,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" ) @@ -68,7 +68,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } arch := config.GetArch() - remote, err := ocizarf.NewZarfOrasRemote(pkgSrc, oci.MultiOSPlatformForArch(arch)) + remote, err := zoci.NewZarfOrasRemote(pkgSrc, oci.MultiOSPlatformForArch(arch)) if err != nil { return nil, err } diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 70c6d15e95..b2db7b9b68 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -15,9 +15,9 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -31,7 +31,7 @@ var ( // OCISource is a package source for OCI registries. type OCISource struct { *types.ZarfPackageOptions - *ocizarf.ZarfOrasRemote + *zoci.ZarfOrasRemote } // LoadPackage loads a package from an OCI registry. @@ -119,7 +119,7 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er func (s *OCISource) LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, skipValidation bool) (err error) { var pkg types.ZarfPackage - toPull := ocizarf.PackageAlwaysPull + toPull := zoci.PackageAlwaysPull if wantSBOM { toPull = append(toPull, layout.SBOMTar) } @@ -201,7 +201,7 @@ func (s *OCISource) Collect(dir string) (string, error) { spinner.Success() // TODO (@Noxsios) remove the suffix check at v1.0.0 - isSkeleton := pkg.Build.Architecture == "skeleton" || strings.HasSuffix(s.Repo().Reference.Reference, ocizarf.SkeletonArch) + isSkeleton := pkg.Build.Architecture == "skeleton" || strings.HasSuffix(s.Repo().Reference.Reference, zoci.SkeletonArch) name := NameFromMetadata(&pkg, isSkeleton) dstTarball := filepath.Join(dir, name) diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index 00006a1a9b..e5c0e5fc6e 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -14,8 +14,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" ) @@ -142,7 +142,7 @@ func (s *TarballSource) LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM b } } - toExtract := ocizarf.PackageAlwaysPull + toExtract := zoci.PackageAlwaysPull if wantSBOM { toExtract = append(toExtract, layout.SBOMTar) } diff --git a/src/pkg/ocizarf/common.go b/src/pkg/zoci/common.go similarity index 93% rename from src/pkg/ocizarf/common.go rename to src/pkg/zoci/common.go index adb84418ba..349c2bc284 100644 --- a/src/pkg/ocizarf/common.go +++ b/src/pkg/zoci/common.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. -package ocizarf +// Package zoci contains functions for interacting with Zarf packages stored in OCI registries. +package zoci import ( "github.com/defenseunicorns/zarf/src/config" diff --git a/src/pkg/ocizarf/copier.go b/src/pkg/zoci/copier.go similarity index 88% rename from src/pkg/ocizarf/copier.go rename to src/pkg/zoci/copier.go index dda63ae5ee..7453620b21 100644 --- a/src/pkg/ocizarf/copier.go +++ b/src/pkg/zoci/copier.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. -package ocizarf +// Package zoci contains functions for interacting with Zarf packages stored in OCI registries. +package zoci import ( "context" diff --git a/src/pkg/ocizarf/fetch.go b/src/pkg/zoci/fetch.go similarity index 89% rename from src/pkg/ocizarf/fetch.go rename to src/pkg/zoci/fetch.go index e80478a268..f0f035389f 100644 --- a/src/pkg/ocizarf/fetch.go +++ b/src/pkg/zoci/fetch.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. -package ocizarf +// Package zoci contains functions for interacting with Zarf packages stored in OCI registries. +package zoci import ( "context" diff --git a/src/pkg/ocizarf/pull.go b/src/pkg/zoci/pull.go similarity index 98% rename from src/pkg/ocizarf/pull.go rename to src/pkg/zoci/pull.go index dcb000ec87..0259e846d6 100644 --- a/src/pkg/ocizarf/pull.go +++ b/src/pkg/zoci/pull.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. -package ocizarf +// Package zoci contains functions for interacting with Zarf packages stored in OCI registries. +package zoci import ( "context" diff --git a/src/pkg/ocizarf/push.go b/src/pkg/zoci/push.go similarity index 95% rename from src/pkg/ocizarf/push.go rename to src/pkg/zoci/push.go index 64bdb05ab9..189fa93580 100644 --- a/src/pkg/ocizarf/push.go +++ b/src/pkg/zoci/push.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. -package ocizarf +// Package zoci contains functions for interacting with Zarf packages stored in OCI registries. +package zoci import ( "context" diff --git a/src/pkg/ocizarf/utils.go b/src/pkg/zoci/utils.go similarity index 92% rename from src/pkg/ocizarf/utils.go rename to src/pkg/zoci/utils.go index 5f10b13c18..2683ef6ab5 100644 --- a/src/pkg/ocizarf/utils.go +++ b/src/pkg/zoci/utils.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package ocizarf contains functions for interacting with Zarf packages stored in OCI registries. -package ocizarf +// Package zoci contains functions for interacting with Zarf packages stored in OCI registries. +package zoci import ( "errors" diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 88ae91295c..1f6f84678c 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/ocizarf" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "oras.land/oras-go/v2/registry" @@ -131,10 +131,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { defer e2e.TeardownRegistry(t, dstRegistryPort) config.CommonOptions.Insecure = true - src, err := ocizarf.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(e2e.Arch)) + src, err := zoci.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(e2e.Arch)) suite.NoError(err) - dst, err := ocizarf.NewZarfOrasRemote(dstRef, oci.MultiOSPlatformForArch(e2e.Arch)) + dst, err := zoci.NewZarfOrasRemote(dstRef, oci.MultiOSPlatformForArch(e2e.Arch)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) @@ -152,7 +152,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { } require.Less(t, attempt, 5, "failed to ping registry") - err = ocizarf.CopyPackage(ctx, src, dst, nil, 5) + err = zoci.CopyPackage(ctx, src, dst, nil, 5) suite.NoError(err) srcRoot, err := src.FetchRoot(ctx) From 957f99a2fa0117c621d710b165db2531cf120a40 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 1 Feb 2024 20:23:21 +0000 Subject: [PATCH 038/113] refactors --- src/internal/packager/images/pull.go | 1 - src/pkg/message/message.go | 1 - src/pkg/oci/common.go | 4 +--- src/pkg/oci/manifest.go | 2 +- src/pkg/oci/push.go | 4 ++-- src/pkg/zoci/common.go | 4 ++-- src/pkg/zoci/push.go | 2 +- src/test/e2e/50_oci_publish_deploy_test.go | 6 ++---- 8 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 01913a0040..5b7f4c1c51 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -74,7 +74,6 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { spinner.Updatef("Fetching image metadata (0 of %d)", len(i.ImageList)) // Spawn a goroutine for each image to load its metadata - // I would like to understand this code better for _, refInfo := range i.ImageList { // Create a closure so that we can pass the src into the goroutine refInfo := refInfo diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index d311995fed..83cf9cc874 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -167,7 +167,6 @@ func Debug(payload ...any) { func Debugf(format string, a ...any) { message := fmt.Sprintf(format, a...) debugPrinter(2, message) - } // ErrorWebf prints an error message and returns a web response. diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index cdeb32851f..ba3b14f9fb 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -106,8 +106,6 @@ func WithMediaType(mediaType string) Modifier { } // MultiOSPlatformForArch sets the target architecture for the remote -// ?! Do we want to expose this function or put it in the zarf specific land ? -// Same question for the MultiOS variable func MultiOSPlatformForArch(arch string) ocispec.Platform { return ocispec.Platform{ OS: MultiOS, @@ -124,7 +122,7 @@ func WithUserAgent(userAgent string) Modifier { // NewOrasRemote returns an oras remote repository client and context for the given url. // -// # Registry auth is handled by the Docker CLI's credential store and checked before returning the client +// Registry auth is handled by the Docker CLI's credential store and checked before returning the client func NewOrasRemote(url string, logger Logger, platform ocispec.Platform, mods ...Modifier) (*OrasRemote, error) { ref, err := registry.ParseReference(strings.TrimPrefix(url, helpers.OCIURLPrefix)) if err != nil { diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index 0f6fce395e..527e71d5d6 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -20,7 +20,7 @@ type Manifest struct { ocispec.Manifest } -// New returns a new OCIManifest +// New returns a new Manifest func New(manifest *ocispec.Manifest) *Manifest { return &Manifest{*manifest} } diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 2610c438f9..7211edaea3 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -76,8 +76,8 @@ func (o *OrasRemote) generatePackManifest(ctx context.Context, src *file.Store, return root, nil } -// PublishPackage publishes the package to the remote repository. -func (o *OrasRemote) PublishPackage(ctx context.Context, src *file.Store, annotations map[string]string, +// PublishArtifact publishes the artifact to the remote repository. +func (o *OrasRemote) PublishArtifact(ctx context.Context, src *file.Store, annotations map[string]string, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index 349c2bc284..08fd341be8 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -28,7 +28,7 @@ type ZarfOrasRemote struct { // NewZarfOrasRemote returns an oras remote repository client and context for the given url // with zarf opination embedded func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { - modifiers := append(mod, oci.WithMediaType(ZarfConfigMediaType), oci.WithInsecure(config.CommonOptions.Insecure)) + modifiers := append([]oci.Modifier{oci.WithMediaType(ZarfConfigMediaType), oci.WithInsecure(config.CommonOptions.Insecure)}, mod...) remote, err := oci.NewOrasRemote(url, &message.Logger{}, platform, modifiers...) if err != nil { return nil, err @@ -36,7 +36,7 @@ func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifie return &ZarfOrasRemote{remote}, nil } -// PlatformForSkeleton returns a skeleton oci +// PlatformForSkeleton sets the target architecture for the remote to skeleton func PlatformForSkeleton() ocispec.Platform { return ocispec.Platform{ OS: oci.MultiOS, diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 189fa93580..3516431310 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -53,7 +53,7 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) annotations := zarfPackageOciAnnotations(&pkg.Metadata) - err = o.PublishPackage(ctx, src, annotations, descs, config.CommonOptions.OCIConcurrency, progressBar) + err = o.PublishArtifact(ctx, src, annotations, descs, config.CommonOptions.OCIConcurrency, progressBar) if err != nil { progressBar.Stop() return fmt.Errorf("unable to publish package: %w", err) diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 1f6f84678c..50d47fc600 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/stretchr/testify/require" @@ -130,11 +129,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - config.CommonOptions.Insecure = true - src, err := zoci.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(e2e.Arch)) + src, err := zoci.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) - dst, err := zoci.NewZarfOrasRemote(dstRef, oci.MultiOSPlatformForArch(e2e.Arch)) + dst, err := zoci.NewZarfOrasRemote(dstRef, oci.MultiOSPlatformForArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) From b342d4155709d9d45d55c527542db276960aee14 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Fri, 2 Feb 2024 09:12:36 -0500 Subject: [PATCH 039/113] Apply suggestions from code review Co-authored-by: razzle --- src/pkg/oci/copier.go | 3 +-- src/pkg/oci/manifest.go | 4 ++-- src/pkg/packager/composer/oci.go | 3 +-- src/pkg/zoci/push.go | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 075736cf20..8eb5c31d9a 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -52,8 +52,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, } if exists { src.log.Debug("Layer already exists in destination, skipping") - contentLength := layer.Size - data := make([]byte, contentLength) + data := make([]byte, layer.Size) progressBar.Write(data) progressBar.UpdateTitle(fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) sem.Release(1) diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index 527e71d5d6..c3e9cea969 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -38,8 +38,8 @@ func (m *Manifest) MarshalJSON() ([]byte, error) { return json.Marshal(m.Manifest) } -// GetLayers returns all the layers in the manifest -func (m *Manifest) GetLayers(include func(d ocispec.Descriptor) bool) []ocispec.Descriptor { +// Filter returns all the layers in the manifest satisfying `include` +func (m *Manifest) Filter(include func(d ocispec.Descriptor) bool) []ocispec.Descriptor { var layers []ocispec.Descriptor for _, layer := range m.Layers { if include != nil && include(layer) { diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index c61e45cdc8..20bc27444e 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -103,8 +103,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { copyOpts.FindSuccessors = content.Successors doneSaving := make(chan error) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) - layerSize := oci.SumDescsSize([]ocispec.Descriptor{componentDesc}) - go utils.RenderProgressBarForLocalDirWrite(cache, layerSize, doneSaving, "Pulling", successText) + go utils.RenderProgressBarForLocalDirWrite(cache, componentDesc.Size, doneSaving, "Pulling", successText) if err := remote.CopyWithProgress(ctx, []ocispec.Descriptor{componentDesc}, store, copyOpts, doneSaving); err != nil { return err } diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 3516431310..9856ff5d3e 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -63,7 +63,7 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf return nil } -func zarfPackageOciAnnotations(metadata *types.ZarfMetadata) map[string]string { +func annotationsFromMetadata(metadata *types.ZarfMetadata) map[string]string { annotations := map[string]string{ ocispec.AnnotationTitle: metadata.Name, From 29c7b55b5f4510a675b02c12312392f98c309adf Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 14:32:34 +0000 Subject: [PATCH 040/113] refactors --- src/pkg/oci/copier.go | 2 +- src/pkg/zoci/copier.go | 2 +- src/pkg/zoci/push.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 8eb5c31d9a..d6d233fa69 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -31,7 +31,7 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return err } - layers := srcRoot.GetLayers(include) + layers := srcRoot.Filter(include) start := time.Now() diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index 7453620b21..67c87d6c5e 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -20,7 +20,7 @@ func CopyPackage(ctx context.Context, src *ZarfOrasRemote, dst *ZarfOrasRemote, if err != nil { return err } - layers := srcRoot.GetLayers(include) + layers := srcRoot.Filter(include) size := oci.SumDescsSize(srcRoot.Layers) title := fmt.Sprintf("[0/%d] layers copied", len(layers)) diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 9856ff5d3e..58cf8831c5 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -52,7 +52,7 @@ func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.Zarf } progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) - annotations := zarfPackageOciAnnotations(&pkg.Metadata) + annotations := annotationsFromMetadata(&pkg.Metadata) err = o.PublishArtifact(ctx, src, annotations, descs, config.CommonOptions.OCIConcurrency, progressBar) if err != nil { progressBar.Stop() From 21f5e447d492a92d279d5b747fee05e70373897a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 14:33:42 +0000 Subject: [PATCH 041/113] refactor --- src/cmd/initialize.go | 4 ++-- src/cmd/tools/zarf.go | 4 ++-- src/pkg/packager/composer/list.go | 2 +- src/pkg/packager/composer/oci.go | 4 ++-- src/pkg/packager/create_stages.go | 4 ++-- src/pkg/packager/publish.go | 6 +++--- src/pkg/packager/sources/new.go | 2 +- src/pkg/packager/sources/oci.go | 2 +- src/pkg/zoci/common.go | 10 +++++----- src/pkg/zoci/copier.go | 2 +- src/pkg/zoci/fetch.go | 4 ++-- src/pkg/zoci/pull.go | 8 ++++---- src/pkg/zoci/push.go | 2 +- src/test/e2e/50_oci_publish_deploy_test.go | 4 ++-- 14 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 81bb53751b..0b4cee2fa7 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -139,11 +139,11 @@ func downloadInitPackage(cacheDirectory string) (string, error) { // If the user wants to download the init-package, download it if confirmDownload { - remote, err := zoci.NewZarfOrasRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { return "", err } - source := &sources.OCISource{ZarfOrasRemote: remote} + source := &sources.OCISource{Remote: remote} return source.Collect(cacheDirectory) } // Otherwise, exit and tell the user to manually download the init-package diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index 4e81f01d4f..2470392da2 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -184,12 +184,12 @@ var downloadInitCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { url := zoci.GetInitPackageURL(config.CLIVersion) - remote, err := zoci.NewZarfOrasRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } - source := &sources.OCISource{ZarfOrasRemote: remote} + source := &sources.OCISource{Remote: remote} _, err = source.Collect(outputDirectory) if err != nil { diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 07297ea306..96d38e891a 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -81,7 +81,7 @@ type ImportChain struct { head *Node tail *Node - remote *zoci.ZarfOrasRemote + remote *zoci.Remote } // Head returns the first node in the import chain diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 20bc27444e..a234b152ab 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -24,12 +24,12 @@ import ( ocistore "oras.land/oras-go/v2/content/oci" ) -func (ic *ImportChain) getRemote(url string) (*zoci.ZarfOrasRemote, error) { +func (ic *ImportChain) getRemote(url string) (*zoci.Remote, error) { if ic.remote != nil { return ic.remote, nil } var err error - ic.remote, err = zoci.NewZarfOrasRemote(url, zoci.PlatformForSkeleton()) + ic.remote, err = zoci.NewRemote(url, zoci.PlatformForSkeleton()) if err != nil { return nil, err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 22b0536a5f..05354083a2 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -252,7 +252,7 @@ func (p *Packager) output() error { if err != nil { return err } - remote, err := zoci.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(ref, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { return err } @@ -655,7 +655,7 @@ func (p *Packager) loadDifferentialData() error { // Load the package spec of the package we're using as a 'reference' for the differential build if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := zoci.NewZarfOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.MultiOSPlatformForArch(config.GetArch())) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index dadf3299cf..f0a1fa2313 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -30,7 +30,7 @@ func (p *Packager) Publish() (err error) { ctx := context.TODO() // oci --> oci is a special case, where we will use oci.CopyPackage so that we can transfer the package // w/o layers touching the filesystem - srcRemote := p.source.(*sources.OCISource).ZarfOrasRemote + srcRemote := p.source.(*sources.OCISource).Remote parts := strings.Split(srcRemote.Repo().Reference.Repository, "/") packageName := parts[len(parts)-1] @@ -38,7 +38,7 @@ func (p *Packager) Publish() (err error) { p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName arch := config.GetArch() - dstRemote, err := zoci.NewZarfOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.MultiOSPlatformForArch(arch)) + dstRemote, err := zoci.NewRemote(p.cfg.PublishOpts.PackageDestination, oci.MultiOSPlatformForArch(arch)) if err != nil { return err } @@ -118,7 +118,7 @@ func (p *Packager) Publish() (err error) { } else { platform = oci.MultiOSPlatformForArch(config.GetArch()) } - remote, err := zoci.NewZarfOrasRemote(ref, platform) + remote, err := zoci.NewRemote(ref, platform) if err != nil { return err } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index bf8a248203..025e26cd6c 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -68,7 +68,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } arch := config.GetArch() - remote, err := zoci.NewZarfOrasRemote(pkgSrc, oci.MultiOSPlatformForArch(arch)) + remote, err := zoci.NewRemote(pkgSrc, oci.MultiOSPlatformForArch(arch)) if err != nil { return nil, err } diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index b2db7b9b68..5e2dbf5448 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -31,7 +31,7 @@ var ( // OCISource is a package source for OCI registries. type OCISource struct { *types.ZarfPackageOptions - *zoci.ZarfOrasRemote + *zoci.Remote } // LoadPackage loads a package from an OCI registry. diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index 08fd341be8..10d9b0333d 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -20,20 +20,20 @@ var ( SkeletonArch = "skeleton" ) -// ZarfOrasRemote is a wrapper around the Oras remote repository with zarf specific functions -type ZarfOrasRemote struct { +// Remote is a wrapper around the Oras remote repository with zarf specific functions +type Remote struct { *oci.OrasRemote } -// NewZarfOrasRemote returns an oras remote repository client and context for the given url +// NewRemote returns an oras remote repository client and context for the given url // with zarf opination embedded -func NewZarfOrasRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*ZarfOrasRemote, error) { +func NewRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*Remote, error) { modifiers := append([]oci.Modifier{oci.WithMediaType(ZarfConfigMediaType), oci.WithInsecure(config.CommonOptions.Insecure)}, mod...) remote, err := oci.NewOrasRemote(url, &message.Logger{}, platform, modifiers...) if err != nil { return nil, err } - return &ZarfOrasRemote{remote}, nil + return &Remote{remote}, nil } // PlatformForSkeleton sets the target architecture for the remote to skeleton diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index 67c87d6c5e..779629bd91 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -14,7 +14,7 @@ import ( ) // CopyPackage copies a zarf package from one OCI registry to another -func CopyPackage(ctx context.Context, src *ZarfOrasRemote, dst *ZarfOrasRemote, include func(d ocispec.Descriptor) bool, concurrency int) error { +func CopyPackage(ctx context.Context, src *Remote, dst *Remote, include func(d ocispec.Descriptor) bool, concurrency int) error { srcRoot, err := src.FetchRoot(ctx) if err != nil { diff --git a/src/pkg/zoci/fetch.go b/src/pkg/zoci/fetch.go index f0f035389f..8d3c27d74e 100644 --- a/src/pkg/zoci/fetch.go +++ b/src/pkg/zoci/fetch.go @@ -14,7 +14,7 @@ import ( ) // FetchZarfYAML fetches the zarf.yaml file from the remote repository. -func (o *ZarfOrasRemote) FetchZarfYAML(ctx context.Context) (pkg types.ZarfPackage, err error) { +func (o *Remote) FetchZarfYAML(ctx context.Context) (pkg types.ZarfPackage, err error) { manifest, err := o.FetchRoot(ctx) if err != nil { return pkg, err @@ -23,7 +23,7 @@ func (o *ZarfOrasRemote) FetchZarfYAML(ctx context.Context) (pkg types.ZarfPacka } // FetchImagesIndex fetches the images/index.json file from the remote repository. -func (o *ZarfOrasRemote) FetchImagesIndex(ctx context.Context) (index *ocispec.Index, err error) { +func (o *Remote) FetchImagesIndex(ctx context.Context) (index *ocispec.Index, err error) { manifest, err := o.FetchRoot(ctx) if err != nil { return index, err diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index 0259e846d6..7fc35afff3 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -39,7 +39,7 @@ var ( // - zarf.yaml // - checksums.txt // - zarf.yaml.sig -func (o *ZarfOrasRemote) PullPackage(ctx context.Context, destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { +func (o *Remote) PullPackage(ctx context.Context, destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { isPartialPull := len(layersToPull) > 0 message.Debugf("Pulling %s", o.Repo().Reference) @@ -72,7 +72,7 @@ func (o *ZarfOrasRemote) PullPackage(ctx context.Context, destinationDir string, // It also retrieves the descriptors for all image layers that are required by the components. // // It also respects the `required` flag on components, and will retrieve all necessary layers for required components. -func (o *ZarfOrasRemote) LayersFromRequestedComponents(ctx context.Context, requestedComponents []string) (layers []ocispec.Descriptor, err error) { +func (o *Remote) LayersFromRequestedComponents(ctx context.Context, requestedComponents []string) (layers []ocispec.Descriptor, err error) { root, err := o.FetchRoot(ctx) if err != nil { return nil, err @@ -151,11 +151,11 @@ func (o *ZarfOrasRemote) LayersFromRequestedComponents(ctx context.Context, requ } // PullPackageMetadata pulls the package metadata from the remote repository and saves it to `destinationDir`. -func (o *ZarfOrasRemote) PullPackageMetadata(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { +func (o *Remote) PullPackageMetadata(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { return o.PullFilesAtPaths(ctx, PackageAlwaysPull, destinationDir) } // PullPackageSBOM pulls the package's sboms.tar from the remote repository and saves it to `destinationDir`. -func (o *ZarfOrasRemote) PullPackageSBOM(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { +func (o *Remote) PullPackageSBOM(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { return o.PullFilesAtPaths(ctx, []string{layout.SBOMTar}, destinationDir) } diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 58cf8831c5..38803878d8 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -18,7 +18,7 @@ import ( ) // PublishZarfPackage publishes the zarf package to the remote repository. -func (o *ZarfOrasRemote) PublishZarfPackage(ctx context.Context, pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int) error { +func (o *Remote) PublishZarfPackage(ctx context.Context, pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int) error { src, err := file.New(paths.Base) if err != nil { return err diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 50d47fc600..8429910dc6 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -129,10 +129,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - src, err := zoci.NewZarfOrasRemote(ref, oci.MultiOSPlatformForArch(e2e.Arch), oci.WithInsecure(true)) + src, err := zoci.NewRemote(ref, oci.MultiOSPlatformForArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) - dst, err := zoci.NewZarfOrasRemote(dstRef, oci.MultiOSPlatformForArch(e2e.Arch), oci.WithInsecure(true)) + dst, err := zoci.NewRemote(dstRef, oci.MultiOSPlatformForArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) From 8295a9a9e210799d7cf691cf125d344b4c22997d Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 15:57:30 +0000 Subject: [PATCH 042/113] changing modifier order in newremote --- src/cmd/initialize.go | 2 +- src/cmd/tools/zarf.go | 2 +- src/pkg/oci/common.go | 54 ++++++++++++++++------ src/pkg/packager/create_stages.go | 4 +- src/pkg/packager/publish.go | 4 +- src/pkg/packager/sources/new.go | 2 +- src/pkg/zoci/common.go | 8 +++- src/test/e2e/50_oci_publish_deploy_test.go | 4 +- 8 files changed, 55 insertions(+), 25 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 0b4cee2fa7..d01f753a60 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -139,7 +139,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { // If the user wants to download the init-package, download it if confirmDownload { - remote, err := zoci.NewRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(url, oci.PlatformForArch(config.GetArch())) if err != nil { return "", err } diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index 2470392da2..f8b061c8f7 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -184,7 +184,7 @@ var downloadInitCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { url := zoci.GetInitPackageURL(config.CLIVersion) - remote, err := zoci.NewRemote(url, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(url, oci.PlatformForArch(config.GetArch())) if err != nil { message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index ba3b14f9fb..2765196e11 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -25,6 +25,20 @@ const ( MultiOS = "multi" ) +// Debug does nothing +func (l *DiscardLogger) Debug(_ string, _ ...any) {} + +// Info does nothing +func (l *DiscardLogger) Info(_ string, _ ...any) {} + +// Warn does nothing +func (l *DiscardLogger) Warn(_ string, _ ...any) {} + +// Error does nothing +func (l *DiscardLogger) Error(_ string, _ ...any) {} + +type DiscardLogger struct{} + // Logger is an interface built to type Logger interface { Debug(msg string, args ...any) @@ -105,8 +119,8 @@ func WithMediaType(mediaType string) Modifier { } } -// MultiOSPlatformForArch sets the target architecture for the remote -func MultiOSPlatformForArch(arch string) ocispec.Platform { +// PlatformForArch sets the target architecture for the remote +func PlatformForArch(arch string) ocispec.Platform { return ocispec.Platform{ OS: MultiOS, Architecture: arch, @@ -120,17 +134,34 @@ func WithUserAgent(userAgent string) Modifier { } } +// WithUserAgent sets the logger for the remote +func WithLogger(logger Logger) Modifier { + return func(o *OrasRemote) { + o.log = logger + } +} + // NewOrasRemote returns an oras remote repository client and context for the given url. // // Registry auth is handled by the Docker CLI's credential store and checked before returning the client -func NewOrasRemote(url string, logger Logger, platform ocispec.Platform, mods ...Modifier) (*OrasRemote, error) { +func NewOrasRemote(url string, platform ocispec.Platform, mods ...Modifier) (*OrasRemote, error) { ref, err := registry.ParseReference(strings.TrimPrefix(url, helpers.OCIURLPrefix)) if err != nil { return nil, fmt.Errorf("failed to parse OCI reference %q: %w", url, err) } - o := &OrasRemote{} - o.log = logger - o.targetPlatform = &platform + transport := http.DefaultTransport.(*http.Transport).Clone() + o := &OrasRemote{ + repo: &remote.Repository{}, + Transport: helpers.NewTransport(transport, nil), + targetPlatform: &platform} + + for _, mod := range mods { + mod(o) + } + + if o.log == nil { + o.log = &DiscardLogger{} + } if err := o.setRepository(ref); err != nil { return nil, err @@ -141,10 +172,6 @@ func NewOrasRemote(url string, logger Logger, platform ocispec.Platform, mods .. copyOpts.PostCopy = o.printLayerCopied o.CopyOpts = copyOpts - for _, mod := range mods { - mod(o) - } - return o, nil } @@ -175,8 +202,9 @@ func (o *OrasRemote) setRepository(ref registry.Reference) error { if err != nil { return err } - repo.Client = client - o.repo = repo + + o.repo.Reference = repo.Reference + o.repo.Client = client return nil } @@ -187,8 +215,6 @@ func (o *OrasRemote) setRepository(ref registry.Reference) error { // // TODO: instead of using Docker's cred store, should use the new one from ORAS to remove that dep func (o *OrasRemote) createAuthClient(ref registry.Reference) (*auth.Client, error) { - transport := http.DefaultTransport.(*http.Transport).Clone() - o.Transport = helpers.NewTransport(transport, nil) client := &auth.Client{ Cache: auth.DefaultCache, diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 05354083a2..2792762dad 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -252,7 +252,7 @@ func (p *Packager) output() error { if err != nil { return err } - remote, err := zoci.NewRemote(ref, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(ref, oci.PlatformForArch(config.GetArch())) if err != nil { return err } @@ -655,7 +655,7 @@ func (p *Packager) loadDifferentialData() error { // Load the package spec of the package we're using as a 'reference' for the differential build if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := zoci.NewRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.MultiOSPlatformForArch(config.GetArch())) + remote, err := zoci.NewRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch())) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index f0a1fa2313..affdd0dd1a 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -38,7 +38,7 @@ func (p *Packager) Publish() (err error) { p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName arch := config.GetArch() - dstRemote, err := zoci.NewRemote(p.cfg.PublishOpts.PackageDestination, oci.MultiOSPlatformForArch(arch)) + dstRemote, err := zoci.NewRemote(p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch)) if err != nil { return err } @@ -116,7 +116,7 @@ func (p *Packager) Publish() (err error) { if p.cfg.CreateOpts.IsSkeleton { platform = zoci.PlatformForSkeleton() } else { - platform = oci.MultiOSPlatformForArch(config.GetArch()) + platform = oci.PlatformForArch(config.GetArch()) } remote, err := zoci.NewRemote(ref, platform) if err != nil { diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 025e26cd6c..007fef5532 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -68,7 +68,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } arch := config.GetArch() - remote, err := zoci.NewRemote(pkgSrc, oci.MultiOSPlatformForArch(arch)) + remote, err := zoci.NewRemote(pkgSrc, oci.PlatformForArch(arch)) if err != nil { return nil, err } diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index 10d9b0333d..ae7bf8208c 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -28,8 +28,12 @@ type Remote struct { // NewRemote returns an oras remote repository client and context for the given url // with zarf opination embedded func NewRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*Remote, error) { - modifiers := append([]oci.Modifier{oci.WithMediaType(ZarfConfigMediaType), oci.WithInsecure(config.CommonOptions.Insecure)}, mod...) - remote, err := oci.NewOrasRemote(url, &message.Logger{}, platform, modifiers...) + modifiers := append([]oci.Modifier{ + oci.WithMediaType(ZarfConfigMediaType), + oci.WithInsecure(config.CommonOptions.Insecure), + oci.WithLogger(&message.Logger{}), + }, mod...) + remote, err := oci.NewOrasRemote(url, platform, modifiers...) if err != nil { return nil, err } diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 8429910dc6..5c707db678 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -129,10 +129,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - src, err := zoci.NewRemote(ref, oci.MultiOSPlatformForArch(e2e.Arch), oci.WithInsecure(true)) + src, err := zoci.NewRemote(ref, oci.PlatformForArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) - dst, err := zoci.NewRemote(dstRef, oci.MultiOSPlatformForArch(e2e.Arch), oci.WithInsecure(true)) + dst, err := zoci.NewRemote(dstRef, oci.PlatformForArch(e2e.Arch), oci.WithInsecure(true)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) From 2f53378e2caf1a6bba17427164d59a102a7079d4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 16:15:45 +0000 Subject: [PATCH 043/113] refactor --- src/pkg/oci/pull.go | 4 ++-- src/pkg/packager/sources/oci.go | 2 +- src/pkg/zoci/common.go | 2 +- src/pkg/zoci/pull.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 8fe7450f0f..3268dd3b91 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -141,8 +141,8 @@ func (o *OrasRemote) PullLayer(ctx context.Context, desc ocispec.Descriptor, des return helpers.WriteFile(filepath.Join(destinationDir, rel), b) } -// PullFilesAtPaths pulls multiple files from the remote repository and saves them to `destinationDir`. -func (o *OrasRemote) PullFilesAtPaths(ctx context.Context, paths []string, destinationDir string) ([]ocispec.Descriptor, error) { +// PullFilesTodirectory pulls multiple files from the remote repository and saves them to `destinationDir`. +func (o *OrasRemote) PullFilesTodirectory(ctx context.Context, paths []string, destinationDir string) ([]ocispec.Descriptor, error) { paths = helpers.Unique(paths) root, err := o.FetchRoot(ctx) if err != nil { diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 5e2dbf5448..5860c1520d 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -124,7 +124,7 @@ func (s *OCISource) LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, toPull = append(toPull, layout.SBOMTar) } ctx := context.TODO() - layersFetched, err := s.PullFilesAtPaths(ctx, toPull, dst.Base) + layersFetched, err := s.PullFilesTodirectory(ctx, toPull, dst.Base) if err != nil { return err } diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index ae7bf8208c..475f690ce2 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -11,7 +11,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -var ( +const ( // ZarfConfigMediaType is the media type for the manifest config ZarfConfigMediaType = "application/vnd.zarf.config.v1+json" // ZarfLayerMediaTypeBlob is the media type for all Zarf layers due to the range of possible content diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index 7fc35afff3..1ba33ade45 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -152,10 +152,10 @@ func (o *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom // PullPackageMetadata pulls the package metadata from the remote repository and saves it to `destinationDir`. func (o *Remote) PullPackageMetadata(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullFilesAtPaths(ctx, PackageAlwaysPull, destinationDir) + return o.PullFilesTodirectory(ctx, PackageAlwaysPull, destinationDir) } // PullPackageSBOM pulls the package's sboms.tar from the remote repository and saves it to `destinationDir`. func (o *Remote) PullPackageSBOM(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullFilesAtPaths(ctx, []string{layout.SBOMTar}, destinationDir) + return o.PullFilesTodirectory(ctx, []string{layout.SBOMTar}, destinationDir) } From 81b00044a07cc55d740faa3348656dfe878b88de Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 16:20:36 +0000 Subject: [PATCH 044/113] rename --- src/pkg/packager/create_stages.go | 2 +- src/pkg/packager/publish.go | 2 +- src/pkg/zoci/push.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 2792762dad..fc1818279b 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -258,7 +258,7 @@ func (p *Packager) output() error { } ctx := context.TODO() - err = remote.PublishZarfPackage(ctx, &p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) + err = remote.PublishPackage(ctx, &p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) if err != nil { return fmt.Errorf("unable to publish package: %w", err) } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index affdd0dd1a..0c7f6cfb6b 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -134,7 +134,7 @@ func (p *Packager) Publish() (err error) { // Publish the package/skeleton to the registry ctx := context.TODO() - if err := remote.PublishZarfPackage(ctx, &p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency); err != nil { + if err := remote.PublishPackage(ctx, &p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency); err != nil { return err } if p.cfg.CreateOpts.IsSkeleton { diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 38803878d8..0994516694 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -17,8 +17,8 @@ import ( "oras.land/oras-go/v2/content/file" ) -// PublishZarfPackage publishes the zarf package to the remote repository. -func (o *Remote) PublishZarfPackage(ctx context.Context, pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int) error { +// PublishPackage publishes the zarf package to the remote repository. +func (o *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int) error { src, err := file.New(paths.Base) if err != nil { return err From 1c662872b8acddf2282e01c92988fda05396c930 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 17:07:37 +0000 Subject: [PATCH 045/113] rename --- src/pkg/oci/push.go | 4 ++-- src/pkg/zoci/push.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 7211edaea3..f6826d6149 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -76,8 +76,8 @@ func (o *OrasRemote) generatePackManifest(ctx context.Context, src *file.Store, return root, nil } -// PublishArtifact publishes the artifact to the remote repository. -func (o *OrasRemote) PublishArtifact(ctx context.Context, src *file.Store, annotations map[string]string, +// Publish publishes the artifact to the remote repository. +func (o *OrasRemote) Publish(ctx context.Context, src *file.Store, annotations map[string]string, desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 0994516694..c8fe986be2 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -53,7 +53,7 @@ func (o *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) annotations := annotationsFromMetadata(&pkg.Metadata) - err = o.PublishArtifact(ctx, src, annotations, descs, config.CommonOptions.OCIConcurrency, progressBar) + err = o.Publish(ctx, src, annotations, descs, config.CommonOptions.OCIConcurrency, progressBar) if err != nil { progressBar.Stop() return fmt.Errorf("unable to publish package: %w", err) From ac362503b554d293f4be1506fb76abfcb7d41df3 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 17:13:49 +0000 Subject: [PATCH 046/113] removing with insecure option --- src/pkg/oci/common.go | 10 ---------- src/pkg/zoci/common.go | 3 ++- src/test/e2e/50_oci_publish_deploy_test.go | 4 ++-- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 2765196e11..4e5f192bb4 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -81,16 +81,6 @@ type OrasRemote struct { // Modifier is a function that modifies an OrasRemote type Modifier func(*OrasRemote) -// WithInsecure sets plainHTTP and insecure tls for the remote -func WithInsecure(insecure bool) Modifier { - return func(o *OrasRemote) { - plainHTTPMod := WithPlainHTTP(insecure) - plainHTTPMod(o) - insecureTLSMod := WithInsecureSkipVerify(insecure) - insecureTLSMod(o) - } -} - // WithCopyOpts sets the copy options for the remote func WithCopyOpts(opts oras.CopyOptions) Modifier { return func(o *OrasRemote) { diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index 475f690ce2..596c7db6e9 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -30,7 +30,8 @@ type Remote struct { func NewRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*Remote, error) { modifiers := append([]oci.Modifier{ oci.WithMediaType(ZarfConfigMediaType), - oci.WithInsecure(config.CommonOptions.Insecure), + oci.WithPlainHTTP(config.CommonOptions.Insecure), + oci.WithInsecureSkipVerify(config.CommonOptions.Insecure), oci.WithLogger(&message.Logger{}), }, mod...) remote, err := oci.NewOrasRemote(url, platform, modifiers...) diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 5c707db678..2e44beffcf 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -129,10 +129,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - src, err := zoci.NewRemote(ref, oci.PlatformForArch(e2e.Arch), oci.WithInsecure(true)) + src, err := zoci.NewRemote(ref, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true), oci.WithInsecureSkipVerify(true)) suite.NoError(err) - dst, err := zoci.NewRemote(dstRef, oci.PlatformForArch(e2e.Arch), oci.WithInsecure(true)) + dst, err := zoci.NewRemote(dstRef, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true), oci.WithInsecureSkipVerify(true)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) From 75847ce74dcf44659f713cf6444d18231e2ef9a9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 18:21:07 +0000 Subject: [PATCH 047/113] lint --- src/pkg/oci/common.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 4e5f192bb4..ad3c7af14d 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -37,6 +37,8 @@ func (l *DiscardLogger) Warn(_ string, _ ...any) {} // Error does nothing func (l *DiscardLogger) Error(_ string, _ ...any) {} +// DiscardLogger is the default if the WithLogger modifier is not used +// It discards all logs type DiscardLogger struct{} // Logger is an interface built to @@ -124,7 +126,7 @@ func WithUserAgent(userAgent string) Modifier { } } -// WithUserAgent sets the logger for the remote +// WithLogger sets the logger for the remote func WithLogger(logger Logger) Modifier { return func(o *OrasRemote) { o.log = logger From 552fccc76c5a07bc05dae567bc78bf5d779572ea Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 18:42:11 +0000 Subject: [PATCH 048/113] removing unnecessary var --- src/pkg/oci/pull.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 3268dd3b91..09d39c9abf 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -72,16 +72,16 @@ func (o *OrasRemote) PullLayers(ctx context.Context, destinationDir string, conc // to doneSaving and provides an input to doneSaving after it's done func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Descriptor, store oras.Target, copyOpts oras.CopyOptions, doneSaving chan error) error { - estimatedBytes := int64(0) shas := []string{} for _, layer := range layers { - estimatedBytes += layer.Size if len(layer.Digest.String()) > 0 { shas = append(shas, layer.Digest.Encoded()) } } if copyOpts.FindSuccessors == nil { + // ?! We are providing a default find successor? Is this generally applicable? + // Or should we stick with the ORAS default copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { nodes, err := content.Successors(ctx, fetcher, desc) if err != nil { From 471a0feb1b114bc674d1d31810c06a858d90f684 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 19:58:54 +0000 Subject: [PATCH 049/113] take publish out of oci, make mediatype a parameter --- src/pkg/oci/common.go | 8 -------- src/pkg/oci/push.go | 40 +++++----------------------------------- src/pkg/zoci/common.go | 1 - src/pkg/zoci/push.go | 31 ++++++++++++++++++++++++++++--- 4 files changed, 33 insertions(+), 47 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index ad3c7af14d..8decf705b5 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -77,7 +77,6 @@ type OrasRemote struct { targetPlatform *ocispec.Platform userAgent string log Logger - mediaType string } // Modifier is a function that modifies an OrasRemote @@ -104,13 +103,6 @@ func WithInsecureSkipVerify(insecure bool) Modifier { } } -// WithMediaType sets the mediatype for the remote -func WithMediaType(mediaType string) Modifier { - return func(o *OrasRemote) { - o.mediaType = mediaType - } -} - // PlatformForArch sets the target architecture for the remote func PlatformForArch(arch string) ocispec.Platform { return ocispec.Platform{ diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index f6826d6149..e0f52cb7d3 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -39,8 +39,8 @@ func (o *OrasRemote) PushLayer(ctx context.Context, b []byte, mediaType string) return &desc, o.repo.Push(ctx, desc, bytes.NewReader(b)) } -// pushManifestConfigFromMetadata pushes the manifest config with metadata to the remote repository. -func (o *OrasRemote) pushManifestConfigFromMetadata(ctx context.Context, annotations map[string]string) (*ocispec.Descriptor, error) { +// PushManifestConfigFromMetadata pushes the manifest config with metadata to the remote repository. +func (o *OrasRemote) PushManifestConfigFromMetadata(ctx context.Context, annotations map[string]string, configMediaType string) (*ocispec.Descriptor, error) { if annotations[ocispec.AnnotationTitle] == "" { return nil, fmt.Errorf("invalid annotations: please include value for %q", ocispec.AnnotationTitle) } @@ -54,10 +54,11 @@ func (o *OrasRemote) pushManifestConfigFromMetadata(ctx context.Context, annotat return nil, err } // If Media type is not set it will be set to the default - return o.PushLayer(ctx, manifestConfigBytes, o.mediaType) + return o.PushLayer(ctx, manifestConfigBytes, configMediaType) } -func (o *OrasRemote) generatePackManifest(ctx context.Context, src *file.Store, descs []ocispec.Descriptor, +// GeneratePackManifest generates an OCI Image Manifest based on the given parameters +func (o *OrasRemote) GeneratePackManifest(ctx context.Context, src *file.Store, descs []ocispec.Descriptor, configDesc *ocispec.Descriptor, annotations map[string]string) (ocispec.Descriptor, error) { packOpts := oras.PackManifestOptions{ Layers: descs, @@ -76,37 +77,6 @@ func (o *OrasRemote) generatePackManifest(ctx context.Context, src *file.Store, return root, nil } -// Publish publishes the artifact to the remote repository. -func (o *OrasRemote) Publish(ctx context.Context, src *file.Store, annotations map[string]string, - desc []ocispec.Descriptor, concurrency int, progressBar ProgressWriter) (err error) { - copyOpts := o.CopyOpts - copyOpts.Concurrency = concurrency - // assumes referrers API is not supported since OCI artifact - // media type is not supported - o.repo.SetReferrersCapability(false) - - // push the manifest config - // since this config is so tiny, and the content is not used again - // it is not logged to the progress, but will error if it fails - manifestConfigDesc, err := o.pushManifestConfigFromMetadata(ctx, annotations) - if err != nil { - return err - } - root, err := o.generatePackManifest(ctx, src, desc, manifestConfigDesc, annotations) - if err != nil { - return err - } - - o.Transport.ProgressBar = progressBar - - publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), o.repo, "", copyOpts) - if err != nil { - return err - } - - return o.UpdateIndex(ctx, o.repo.Reference.Reference, publishedDesc) -} - // UpdateIndex updates the index for the given package. func (o *OrasRemote) UpdateIndex(ctx context.Context, tag string, publishedDesc ocispec.Descriptor) error { var index ocispec.Index diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index 596c7db6e9..105cec8fff 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -29,7 +29,6 @@ type Remote struct { // with zarf opination embedded func NewRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*Remote, error) { modifiers := append([]oci.Modifier{ - oci.WithMediaType(ZarfConfigMediaType), oci.WithPlainHTTP(config.CommonOptions.Insecure), oci.WithInsecureSkipVerify(config.CommonOptions.Insecure), oci.WithLogger(&message.Logger{}), diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index c8fe986be2..e7a94e941a 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -8,12 +8,12 @@ import ( "context" "fmt" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" "oras.land/oras-go/v2/content/file" ) @@ -51,9 +51,34 @@ func (o *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat total += desc.Size } - progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) annotations := annotationsFromMetadata(&pkg.Metadata) - err = o.Publish(ctx, src, annotations, descs, config.CommonOptions.OCIConcurrency, progressBar) + + // assumes referrers API is not supported since OCI artifact + // media type is not supported + o.Repo().SetReferrersCapability(false) + + // push the manifest config + // since this config is so tiny, and the content is not used again + // it is not logged to the progress, but will error if it fails + manifestConfigDesc, err := o.PushManifestConfigFromMetadata(ctx, annotations, ZarfConfigMediaType) + if err != nil { + return err + } + root, err := o.GeneratePackManifest(ctx, src, descs, manifestConfigDesc, annotations) + if err != nil { + return err + } + + progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) + o.Transport.ProgressBar = progressBar + + publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), o.Repo(), "", copyOpts) + if err != nil { + return err + } + + o.UpdateIndex(ctx, o.Repo().Reference.Reference, publishedDesc) + if err != nil { progressBar.Stop() return fmt.Errorf("unable to publish package: %w", err) From 561412ec9e30bc9ebcd41109c0d94a1f37e5d155 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 20:16:16 +0000 Subject: [PATCH 050/113] rework user agent --- src/pkg/oci/common.go | 18 +++++------------- src/pkg/zoci/common.go | 1 + 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 8decf705b5..b3b90e5f7d 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -75,7 +75,6 @@ type OrasRemote struct { Transport *helpers.Transport CopyOpts oras.CopyOptions targetPlatform *ocispec.Platform - userAgent string log Logger } @@ -114,7 +113,7 @@ func PlatformForArch(arch string) ocispec.Platform { // WithUserAgent sets the target architecture for the remote func WithUserAgent(userAgent string) Modifier { return func(o *OrasRemote) { - o.userAgent = userAgent + o.repo.Client.(*auth.Client).SetUserAgent(userAgent) } } @@ -134,8 +133,10 @@ func NewOrasRemote(url string, platform ocispec.Platform, mods ...Modifier) (*Or return nil, fmt.Errorf("failed to parse OCI reference %q: %w", url, err) } transport := http.DefaultTransport.(*http.Transport).Clone() + client := auth.DefaultClient + client.Client.Transport = transport o := &OrasRemote{ - repo: &remote.Repository{}, + repo: &remote.Repository{Client: client}, Transport: helpers.NewTransport(transport, nil), targetPlatform: &platform} @@ -200,16 +201,7 @@ func (o *OrasRemote) setRepository(ref registry.Reference) error { // TODO: instead of using Docker's cred store, should use the new one from ORAS to remove that dep func (o *OrasRemote) createAuthClient(ref registry.Reference) (*auth.Client, error) { - client := &auth.Client{ - Cache: auth.DefaultCache, - Client: &http.Client{ - Transport: o.Transport, - }, - } - if o.userAgent != "" { - client.SetUserAgent(o.userAgent) - } - + client := o.repo.Client.(*auth.Client) o.log.Debug("Loading docker config file from default config location: %s for %s", config.Dir(), ref) cfg, err := config.Load(config.Dir()) if err != nil { diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index 105cec8fff..2614c081c1 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -32,6 +32,7 @@ func NewRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*Rem oci.WithPlainHTTP(config.CommonOptions.Insecure), oci.WithInsecureSkipVerify(config.CommonOptions.Insecure), oci.WithLogger(&message.Logger{}), + oci.WithUserAgent("zarf/" + config.CLIVersion), }, mod...) remote, err := oci.NewOrasRemote(url, platform, modifiers...) if err != nil { From d224d2f877c0180d9ce3865bd169182bacea6e87 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 20:32:39 +0000 Subject: [PATCH 051/113] go mod tidy, remove comments --- go.mod | 6 +++--- go.sum | 12 ++++++------ src/pkg/oci/pull.go | 2 -- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index f6037dd23b..0e9a821664 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/gosuri/uitable v0.0.4 github.com/mholt/archiver/v3 v3.5.1 github.com/moby/moby v24.0.7+incompatible - github.com/opencontainers/image-spec v1.1.0-rc5 + github.com/opencontainers/image-spec v1.1.0-rc6 github.com/otiai10/copy v1.14.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 @@ -45,7 +45,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.17.0 - golang.org/x/sync v0.5.0 + golang.org/x/sync v0.6.0 golang.org/x/term v0.16.0 helm.sh/helm/v3 v3.13.2 k8s.io/api v0.28.4 @@ -54,7 +54,7 @@ require ( k8s.io/component-base v0.28.4 k8s.io/klog/v2 v2.110.1 k8s.io/kubectl v0.28.4 - oras.land/oras-go/v2 v2.3.1 + oras.land/oras-go/v2 v2.4.0 sigs.k8s.io/kustomize/api v0.16.0 sigs.k8s.io/kustomize/kyaml v0.16.0 sigs.k8s.io/yaml v1.4.0 diff --git a/go.sum b/go.sum index 880a7aff5c..2155d8a056 100644 --- a/go.sum +++ b/go.sum @@ -1371,8 +1371,8 @@ github.com/open-policy-agent/opa v0.59.0 h1:1WFU/KUhJAr3qatm0Lf8Ea5jp10ZmlE2M07o github.com/open-policy-agent/opa v0.59.0/go.mod h1:rdJSkEc4oQ+0074/3Fsgno5bkPsYxTjU5aLNmMujIvI= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU= +github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1951,8 +1951,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2488,8 +2488,8 @@ modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ= modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= -oras.land/oras-go/v2 v2.3.1 h1:lUC6q8RkeRReANEERLfH86iwGn55lbSWP20egdFHVec= -oras.land/oras-go/v2 v2.3.1/go.mod h1:5AQXVEu1X/FKp1F9DMOb5ZItZBOa0y5dha0yCm4NR9c= +oras.land/oras-go/v2 v2.4.0 h1:i+Wt5oCaMHu99guBD0yuBjdLvX7Lz8ukPbwXdR7uBMs= +oras.land/oras-go/v2 v2.4.0/go.mod h1:osvtg0/ClRq1KkydMAEu/IxFieyjItcsQ4ut4PPF+f8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 09d39c9abf..bce292fb5f 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -80,8 +80,6 @@ func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Desc } if copyOpts.FindSuccessors == nil { - // ?! We are providing a default find successor? Is this generally applicable? - // Or should we stick with the ORAS default copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { nodes, err := content.Successors(ctx, fetcher, desc) if err != nil { From 4d3cee102cd8f49f13a7dddc4a13e07efaf3356a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 2 Feb 2024 20:44:32 +0000 Subject: [PATCH 052/113] simplifying filter --- src/pkg/oci/pull.go | 34 +++++--------------------------- src/pkg/packager/composer/oci.go | 3 --- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index bce292fb5f..c06e5c39e2 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -9,12 +9,9 @@ import ( "os" "path/filepath" - "slices" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/content/file" ) @@ -79,34 +76,13 @@ func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Desc } } - if copyOpts.FindSuccessors == nil { - copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - nodes, err := content.Successors(ctx, fetcher, desc) - if err != nil { - return nil, err - } - if desc.MediaType == ocispec.MediaTypeImageIndex { - manifestDescs := nodes - nodes = []ocispec.Descriptor{} - // expand the manifests - for _, node := range manifestDescs { - manifest, err := o.FetchManifest(ctx, node) - if err != nil { - return nil, err - } - nodes = append(nodes, manifest.Layers...) - nodes = append(nodes, manifest.Config) - } - } - - var ret []ocispec.Descriptor - for _, node := range nodes { - if slices.Contains(shas, node.Digest.Encoded()) { - ret = append(ret, node) - } + copyOpts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { + for _, sha := range shas { + if sha == desc.Digest.Encoded() { + return nil } - return ret, nil } + return oras.SkipNode } _, err := oras.Copy(ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index a234b152ab..72b7144648 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -20,7 +20,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/content" ocistore "oras.land/oras-go/v2/content/oci" ) @@ -99,8 +98,6 @@ func (ic *ImportChain) fetchOCISkeleton() error { return err } else if !exists { copyOpts := remote.CopyOpts - // TODO (@WSTARR): This overrides the FindSuccessors function to no longer filter nodes when pulling which is necessary when caching - once we implement caching more thoroughly we will need to reevaluate this. - copyOpts.FindSuccessors = content.Successors doneSaving := make(chan error) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) go utils.RenderProgressBarForLocalDirWrite(cache, componentDesc.Size, doneSaving, "Pulling", successText) From 4f09a084322e58cf1279503580ee46d129aa00e4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 5 Feb 2024 14:00:09 +0000 Subject: [PATCH 053/113] update dependencies --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 0e9a821664..bf816410f0 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/anchore/clio v0.0.0-20240131202212-9eba61247448 - github.com/anchore/stereoscope v0.0.0-20240118133533-eb656fc71793 + github.com/anchore/stereoscope v0.0.1 github.com/anchore/syft v0.99.0 github.com/derailed/k9s v0.29.1 github.com/distribution/reference v0.5.0 @@ -27,7 +27,7 @@ require ( github.com/google/go-containerregistry v0.17.0 github.com/gosuri/uitable v0.0.4 github.com/mholt/archiver/v3 v3.5.1 - github.com/moby/moby v24.0.7+incompatible + github.com/moby/moby v24.0.9+incompatible github.com/opencontainers/image-spec v1.1.0-rc6 github.com/otiai10/copy v1.14.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 2155d8a056..0372961d58 100644 --- a/go.sum +++ b/go.sum @@ -384,6 +384,8 @@ github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501 h1:AV7qjwM github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501/go.mod h1:Blo6OgJNiYF41ufcgHKkbCKF2MDOMlrqhXv/ij6ocR4= github.com/anchore/stereoscope v0.0.0-20240118133533-eb656fc71793 h1:wji+qdjsV7ooolBwb3faVZnEK3WtY/kcT5473kxVZS4= github.com/anchore/stereoscope v0.0.0-20240118133533-eb656fc71793/go.mod h1:IylG7ofLoUKHwS1XDF6rPhOmaE3GgpAgsMdvvYfooTU= +github.com/anchore/stereoscope v0.0.1 h1:OxF7PaxMltnAxjLnDMyka+SKRIQar/bBkDdavsnjyxM= +github.com/anchore/stereoscope v0.0.1/go.mod h1:IylG7ofLoUKHwS1XDF6rPhOmaE3GgpAgsMdvvYfooTU= github.com/anchore/syft v0.99.0 h1:oqycIA7XfHCB09meroN7eY2RWTGUZIdtWsMQL2HlPvw= github.com/anchore/syft v0.99.0/go.mod h1:tGZGyDxB2z/yu+x266+b67fMenGKCrUvSNVKED1euuo= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= @@ -1298,6 +1300,8 @@ github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/moby v24.0.7+incompatible h1:RrVT5IXBn85mRtFKP+gFwVLCcnNPZIgN3NVRJG9Le+4= github.com/moby/moby v24.0.7+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= +github.com/moby/moby v24.0.9+incompatible h1:Z/hFbZJqC5Fmuf6jesMLdHU71CMAgdiSJ1ZYey+bFmg= +github.com/moby/moby v24.0.9+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= From fa11785f56e3bfa7dbfd2d1b7fde5abab77f01f3 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 5 Feb 2024 18:39:13 +0000 Subject: [PATCH 054/113] reordered function names for api --- src/pkg/oci/pull.go | 8 ++++---- src/pkg/packager/sources/oci.go | 2 +- src/pkg/zoci/pull.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index c06e5c39e2..f73d6e10f6 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -104,7 +104,7 @@ func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Desc } // PullLayer pulls a layer from the remote repository and saves it to `destinationDir/annotationTitle`. -func (o *OrasRemote) PullLayer(ctx context.Context, desc ocispec.Descriptor, destinationDir string) error { +func (o *OrasRemote) PullLayer(ctx context.Context, destinationDir string, desc ocispec.Descriptor) error { b, err := o.FetchLayer(ctx, desc) if err != nil { return err @@ -115,8 +115,8 @@ func (o *OrasRemote) PullLayer(ctx context.Context, desc ocispec.Descriptor, des return helpers.WriteFile(filepath.Join(destinationDir, rel), b) } -// PullFilesTodirectory pulls multiple files from the remote repository and saves them to `destinationDir`. -func (o *OrasRemote) PullFilesTodirectory(ctx context.Context, paths []string, destinationDir string) ([]ocispec.Descriptor, error) { +// PullPaths pulls multiple files from the remote repository and saves them to `destinationDir`. +func (o *OrasRemote) PullPaths(ctx context.Context, destinationDir string, paths []string) ([]ocispec.Descriptor, error) { paths = helpers.Unique(paths) root, err := o.FetchRoot(ctx) if err != nil { @@ -130,7 +130,7 @@ func (o *OrasRemote) PullFilesTodirectory(ctx context.Context, paths []string, d if o.FileDescriptorExists(desc, destinationDir) { continue } - err = o.PullLayer(ctx, desc, destinationDir) + err = o.PullLayer(ctx, destinationDir, desc) if err != nil { return nil, err } diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 5860c1520d..1337fed765 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -124,7 +124,7 @@ func (s *OCISource) LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, toPull = append(toPull, layout.SBOMTar) } ctx := context.TODO() - layersFetched, err := s.PullFilesTodirectory(ctx, toPull, dst.Base) + layersFetched, err := s.PullPaths(ctx, dst.Base, toPull) if err != nil { return err } diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index 1ba33ade45..d5a8516196 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -152,10 +152,10 @@ func (o *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom // PullPackageMetadata pulls the package metadata from the remote repository and saves it to `destinationDir`. func (o *Remote) PullPackageMetadata(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullFilesTodirectory(ctx, PackageAlwaysPull, destinationDir) + return o.PullPaths(ctx, destinationDir, PackageAlwaysPull) } // PullPackageSBOM pulls the package's sboms.tar from the remote repository and saves it to `destinationDir`. func (o *Remote) PullPackageSBOM(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullFilesTodirectory(ctx, []string{layout.SBOMTar}, destinationDir) + return o.PullPaths(ctx, destinationDir, []string{layout.SBOMTar}) } From 40d968fca2f1bfd6b6dacdf83194c78f93d4573e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 5 Feb 2024 19:23:47 +0000 Subject: [PATCH 055/113] comment --- src/pkg/zoci/copier.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index 779629bd91..d808eaa73a 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -14,6 +14,8 @@ import ( ) // CopyPackage copies a zarf package from one OCI registry to another +// TODO (@AustinAbro321) we should see if we should also copy the manifests for a complete copy in this function +// Either through ORAS or additional logic found in callers of this function func CopyPackage(ctx context.Context, src *Remote, dst *Remote, include func(d ocispec.Descriptor) bool, concurrency int) error { srcRoot, err := src.FetchRoot(ctx) From c0acb3ce555903d7304d2fa18e64743a56436632 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 6 Feb 2024 20:49:52 +0000 Subject: [PATCH 056/113] getting rid of finish function --- src/pkg/zoci/copier.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index d808eaa73a..f1a1d9ac9b 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -27,7 +27,11 @@ func CopyPackage(ctx context.Context, src *Remote, dst *Remote, include func(d o title := fmt.Sprintf("[0/%d] layers copied", len(layers)) progressBar := message.NewProgressBar(size, title) - defer progressBar.Finish(err, "Copied %s", src.Repo().Reference) + defer progressBar.Stop() - return oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, include, concurrency, progressBar) + if err := oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, include, concurrency, progressBar); err != nil { + return err + } + progressBar.Successf("Copied %s", src.Repo().Reference) + return nil } From 15f31e0421b1aa5d5815041e5f047b819e6da0d4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 6 Feb 2024 20:50:24 +0000 Subject: [PATCH 057/113] getting rid of finish function --- src/pkg/message/progress.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/pkg/message/progress.go b/src/pkg/message/progress.go index be2da348f7..a6c819093f 100644 --- a/src/pkg/message/progress.go +++ b/src/pkg/message/progress.go @@ -78,15 +78,6 @@ func (p *ProgressBar) Write(data []byte) (int, error) { return n, nil } -// Finish stops the progressbar and logs the approiate message based on if there was an error -func (p *ProgressBar) Finish(err error, format string, a ...any) { - if err != nil { - p.Errorf(err, format, a...) - } else { - p.Successf(format, a...) - } -} - // Successf marks the ProgressBar as successful in the CLI. func (p *ProgressBar) Successf(format string, a ...any) { p.Stop() From e026f4a641654615e0e7cd62397b00a3d1d12dc3 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 6 Feb 2024 21:23:46 +0000 Subject: [PATCH 058/113] Taking channels out of pure oci package --- src/pkg/oci/pull.go | 23 ++++------------------- src/pkg/packager/composer/oci.go | 6 +++++- src/pkg/zoci/pull.go | 5 ++++- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index f73d6e10f6..8e3ccd7919 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -45,10 +45,8 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi } // PullLayers pulls the package from the remote repository and saves it to the given path. -// If you don't set doneSaving to nil you must have a goroutine running that acts on an input -// to doneSaving and provides an input to doneSaving after it's done func (o *OrasRemote) PullLayers(ctx context.Context, destinationDir string, concurrency int, - layersToPull []ocispec.Descriptor, doneSaving chan error) ([]ocispec.Descriptor, error) { + layersToPull []ocispec.Descriptor) ([]ocispec.Descriptor, error) { // de-duplicate layers layersToPull = RemoveDuplicateDescriptors(layersToPull) @@ -61,14 +59,11 @@ func (o *OrasRemote) PullLayers(ctx context.Context, destinationDir string, conc copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - return layersToPull, o.CopyWithProgress(ctx, layersToPull, dst, copyOpts, doneSaving) + return layersToPull, o.CopyToStore(ctx, layersToPull, dst, copyOpts) } -// CopyWithProgress copies the given layers from the remote repository to the given store. -// If you don't set doneSaving to nil you must have a goroutine running that acts on an input -// to doneSaving and provides an input to doneSaving after it's done -func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Descriptor, store oras.Target, - copyOpts oras.CopyOptions, doneSaving chan error) error { +// CopyToStore copies the given layers from the remote repository to the given store. +func (o *OrasRemote) CopyToStore(ctx context.Context, layers []ocispec.Descriptor, store oras.Target, copyOpts oras.CopyOptions) error { shas := []string{} for _, layer := range layers { if len(layer.Digest.String()) > 0 { @@ -87,19 +82,9 @@ func (o *OrasRemote) CopyWithProgress(ctx context.Context, layers []ocispec.Desc _, err := oras.Copy(ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) if err != nil { - if doneSaving != nil { - doneSaving <- err - <-doneSaving - } return err } - // Send a signal to the progress bar that we're done and wait for it to finish - if doneSaving != nil { - doneSaving <- nil - <-doneSaving - } - return nil } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 72b7144648..491f6affd1 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -101,9 +101,13 @@ func (ic *ImportChain) fetchOCISkeleton() error { doneSaving := make(chan error) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) go utils.RenderProgressBarForLocalDirWrite(cache, componentDesc.Size, doneSaving, "Pulling", successText) - if err := remote.CopyWithProgress(ctx, []ocispec.Descriptor{componentDesc}, store, copyOpts, doneSaving); err != nil { + err = remote.CopyToStore(ctx, []ocispec.Descriptor{componentDesc}, store, copyOpts) + doneSaving <- err + <-doneSaving + if err != nil { return err } + } } diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index d5a8516196..1c8beca3c2 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -65,7 +65,10 @@ func (o *Remote) PullPackage(ctx context.Context, destinationDir string, concurr layerSize := oci.SumDescsSize(layersToPull) go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, doneSaving, "Pulling", successText) - return o.PullLayers(ctx, destinationDir, concurrency, layersToPull, doneSaving) + layers, err := o.PullLayers(ctx, destinationDir, concurrency, layersToPull) + doneSaving <- err + <-doneSaving + return layers, err } // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. From f2aff800baf7dd82f2efafec9c0b5e3a5d85e4a9 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Wed, 7 Feb 2024 09:19:15 -0500 Subject: [PATCH 059/113] Apply suggestions from code review Co-authored-by: razzle --- src/pkg/oci/common.go | 4 ++-- src/pkg/oci/manifest.go | 4 ++-- src/pkg/oci/pull.go | 6 ++++++ src/pkg/zoci/common.go | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index b3b90e5f7d..bab25b4c50 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -49,12 +49,12 @@ type Logger interface { Error(msg string, args ...any) } -// Write doesn't do anything but satify implementation +// Write doesn't do anything but satisfy implementation func (DiscardProgressWriter) Write(p []byte) (int, error) { return len(p), nil } -// UpdateTitle doesn't do anything but satify implementation +// UpdateTitle doesn't do anything but satisfy implementation func (DiscardProgressWriter) UpdateTitle(_ string) {} // DiscardProgressWriter is a ProgressWriter in which all calls succeed without doing anything diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index c3e9cea969..e064c18499 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -20,8 +20,8 @@ type Manifest struct { ocispec.Manifest } -// New returns a new Manifest -func New(manifest *ocispec.Manifest) *Manifest { +// NewManifest returns a new Manifest +func NewManifest(manifest *ocispec.Manifest) *Manifest { return &Manifest{*manifest} } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 8e3ccd7919..a923c8f8c7 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -71,7 +71,13 @@ func (o *OrasRemote) CopyToStore(ctx context.Context, layers []ocispec.Descripto } } + preCopy := copyOpts.PreCopy copyOpts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { + if preCopy != nil { + if err := preCopy(ctx, desc); err != nil { + return err + } + } for _, sha := range shas { if sha == desc.Digest.Encoded() { return nil diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index 2614c081c1..d5654ccedf 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -27,13 +27,13 @@ type Remote struct { // NewRemote returns an oras remote repository client and context for the given url // with zarf opination embedded -func NewRemote(url string, platform ocispec.Platform, mod ...oci.Modifier) (*Remote, error) { +func NewRemote(url string, platform ocispec.Platform, mods ...oci.Modifier) (*Remote, error) { modifiers := append([]oci.Modifier{ oci.WithPlainHTTP(config.CommonOptions.Insecure), oci.WithInsecureSkipVerify(config.CommonOptions.Insecure), oci.WithLogger(&message.Logger{}), oci.WithUserAgent("zarf/" + config.CLIVersion), - }, mod...) + }, mods...) remote, err := oci.NewOrasRemote(url, platform, modifiers...) if err != nil { return nil, err From 66b7b1413fde4e92b70dad505f03f0e340f418bf Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 15:00:35 +0000 Subject: [PATCH 060/113] refactor --- src/pkg/oci/copier.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index d6d233fa69..704068ef4a 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -36,11 +36,11 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, start := time.Now() for idx, layer := range layers { - bytes, err := json.MarshalIndent(layer, "", " ") + b, err := json.MarshalIndent(layer, "", " ") if err != nil { src.log.Debug("ERROR marshalling json: %s", err.Error()) } - src.log.Debug("Copying layer:", string(bytes)) + src.log.Debug("Copying layer:", string(b)) if err := sem.Acquire(ctx, 1); err != nil { return err } From 5cf4f4db1428e5752c0c0b9c963e0756c856bbaf Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 15:20:07 +0000 Subject: [PATCH 061/113] refactor --- src/pkg/zoci/push.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index e7a94e941a..9124274787 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -46,10 +47,7 @@ func (o *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - var total int64 - for _, desc := range descs { - total += desc.Size - } + total := oci.SumDescsSize(descs) annotations := annotationsFromMetadata(&pkg.Metadata) From 09d02f36ea55f3d47e04a8809e155d8d6364c944 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 15:21:08 +0000 Subject: [PATCH 062/113] use right layers --- src/pkg/zoci/copier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index f1a1d9ac9b..a0907a0b8a 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -23,7 +23,7 @@ func CopyPackage(ctx context.Context, src *Remote, dst *Remote, include func(d o return err } layers := srcRoot.Filter(include) - size := oci.SumDescsSize(srcRoot.Layers) + size := oci.SumDescsSize(layers) title := fmt.Sprintf("[0/%d] layers copied", len(layers)) progressBar := message.NewProgressBar(size, title) From f81502da8d243d7991d5ef9058c9f70121d6752b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 15:21:58 +0000 Subject: [PATCH 063/113] refactor filter --- src/pkg/oci/manifest.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index e064c18499..4aa5eb790f 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -40,14 +40,14 @@ func (m *Manifest) MarshalJSON() ([]byte, error) { // Filter returns all the layers in the manifest satisfying `include` func (m *Manifest) Filter(include func(d ocispec.Descriptor) bool) []ocispec.Descriptor { + if include == nil { + return append(m.Layers, m.Config) + } var layers []ocispec.Descriptor for _, layer := range m.Layers { - if include != nil && include(layer) { - layers = append(layers, layer) - } else if include == nil { + if include(layer) { layers = append(layers, layer) } } - layers = append(layers, m.Config) - return layers + return append(layers, m.Config) } From 67db37783e90a94739dbe397ac359a939cfb839c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 15:50:24 +0000 Subject: [PATCH 064/113] updating oci version --- src/pkg/oci/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index e0f52cb7d3..16ca2e5363 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -46,7 +46,7 @@ func (o *OrasRemote) PushManifestConfigFromMetadata(ctx context.Context, annotat } manifestConfig := ConfigPartial{ Architecture: o.targetPlatform.Architecture, - OCIVersion: "1.0.1", + OCIVersion: specs.Version, Annotations: annotations, } manifestConfigBytes, err := json.Marshal(manifestConfig) From e2db18cedc1d3923ab1d6e194cb84b716a9d905c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 16:03:55 +0000 Subject: [PATCH 065/113] removing unnecessary writeFile function --- src/extensions/bigbang/bigbang.go | 2 +- src/internal/packager/helm/post-render.go | 4 +-- src/internal/packager/kustomize/build.go | 4 +-- src/pkg/cluster/data.go | 3 +-- src/pkg/oci/pull.go | 2 +- src/pkg/packager/create_stages.go | 2 +- src/pkg/utils/helpers/io.go | 31 ----------------------- src/test/e2e/20_zarf_init_test.go | 4 +-- 8 files changed, 9 insertions(+), 43 deletions(-) delete mode 100644 src/pkg/utils/helpers/io.go diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 571f886e25..e6c4cbbece 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -465,7 +465,7 @@ func addBigBangManifests(YOLO bool, manifestDir string, cfg *extensions.BigBang) return err } - if err := helpers.WriteFile(path, out); err != nil { + if err := os.WriteFile(path, out, 0666); err != nil { return err } diff --git a/src/internal/packager/helm/post-render.go b/src/internal/packager/helm/post-render.go index 60c4aa042b..05ff55f828 100644 --- a/src/internal/packager/helm/post-render.go +++ b/src/internal/packager/helm/post-render.go @@ -15,7 +15,6 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/releaseutil" @@ -62,8 +61,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { } path := filepath.Join(tempDir, "chart.yaml") - // Write the context to a file for processing - if err := helpers.WriteFile(path, renderedManifests.Bytes()); err != nil { + if err := os.WriteFile(path, renderedManifests.Bytes(), 0666); err != nil { return nil, fmt.Errorf("unable to write the post-render file for the helm chart") } diff --git a/src/internal/packager/kustomize/build.go b/src/internal/packager/kustomize/build.go index 9b52c8e629..3ed07ab5c2 100644 --- a/src/internal/packager/kustomize/build.go +++ b/src/internal/packager/kustomize/build.go @@ -6,8 +6,8 @@ package kustomize import ( "fmt" + "os" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "sigs.k8s.io/kustomize/api/krusty" krustytypes "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/filesys" @@ -39,5 +39,5 @@ func Build(path string, destination string, kustomizeAllowAnyDirectory bool) err return fmt.Errorf("problem converting kustomization to yaml: %w", err) } - return helpers.WriteFile(destination, yaml) + return os.WriteFile(destination, yaml, 0666) } diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index 7eb57fa6a3..bf9e27534a 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -18,7 +18,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" corev1 "k8s.io/api/core/v1" ) @@ -29,7 +28,7 @@ func (c *Cluster) HandleDataInjection(wg *sync.WaitGroup, data types.ZarfDataInj defer wg.Done() injectionCompletionMarker := filepath.Join(componentPath.DataInjections, config.GetDataInjectionMarker()) - if err := helpers.WriteFile(injectionCompletionMarker, []byte("🦄")); err != nil { + if err := os.WriteFile(injectionCompletionMarker, []byte("🦄"), 0666); err != nil { message.WarnErrf(err, "Unable to create the data injection completion marker") return } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index a923c8f8c7..7c7e373afe 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -103,7 +103,7 @@ func (o *OrasRemote) PullLayer(ctx context.Context, destinationDir string, desc rel := desc.Annotations[ocispec.AnnotationTitle] - return helpers.WriteFile(filepath.Join(destinationDir, rel), b) + return os.WriteFile(filepath.Join(destinationDir, rel), b, 0666) } // PullPaths pulls multiple files from the remote repository and saves them to `destinationDir`. diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index dc8f1f5994..afece18228 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -654,7 +654,7 @@ func (p *Packager) generatePackageChecksums() (string, error) { // Create the checksums file checksumsFilePath := p.layout.Checksums - if err := helpers.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { + if err := os.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n"), 0666); err != nil { return "", err } diff --git a/src/pkg/utils/helpers/io.go b/src/pkg/utils/helpers/io.go deleted file mode 100644 index 670054fa45..0000000000 --- a/src/pkg/utils/helpers/io.go +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package helpers provides generic helper functions with no external imports -package helpers - -import ( - "fmt" - "os" -) - -// WriteFile writes the given data to the given path. -func WriteFile(path string, data []byte) error { - f, err := os.Create(path) - if err != nil { - return fmt.Errorf("unable to create the file at %s to write the contents: %w", path, err) - } - - _, err = f.Write(data) - if err != nil { - _ = f.Close() - return fmt.Errorf("unable to write the file at %s contents:%w", path, err) - } - - err = f.Close() - if err != nil { - return fmt.Errorf("error saving file %s: %w", path, err) - } - - return nil -} diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index cc94ed643f..3aa27d3e7c 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -7,12 +7,12 @@ package test import ( "encoding/base64" "fmt" + "os" "runtime" "testing" "encoding/json" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" ) @@ -150,7 +150,7 @@ func initWithoutStorageClass(t *testing.T) { storageClassFileName := "storage-class.yaml" - err = helpers.WriteFile(storageClassFileName, []byte(storageClassYaml)) + err = os.WriteFile(storageClassFileName, []byte(storageClassYaml), 0666) require.NoError(t, err) defer e2e.CleanFiles(storageClassFileName) From 79b1c05faf1f19e7d0cd1c5fdb50fb63a667c881 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 16:07:07 +0000 Subject: [PATCH 066/113] switching helpers.writefile to os.writefile --- src/extensions/bigbang/bigbang.go | 2 +- src/internal/packager/helm/post-render.go | 2 +- src/internal/packager/kustomize/build.go | 2 +- src/pkg/cluster/data.go | 2 +- src/pkg/oci/pull.go | 2 +- src/pkg/packager/create_stages.go | 2 +- src/test/e2e/20_zarf_init_test.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index e6c4cbbece..dca401a15b 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -465,7 +465,7 @@ func addBigBangManifests(YOLO bool, manifestDir string, cfg *extensions.BigBang) return err } - if err := os.WriteFile(path, out, 0666); err != nil { + if err := os.WriteFile(path, out, 0644); err != nil { return err } diff --git a/src/internal/packager/helm/post-render.go b/src/internal/packager/helm/post-render.go index 05ff55f828..2a22cf5a50 100644 --- a/src/internal/packager/helm/post-render.go +++ b/src/internal/packager/helm/post-render.go @@ -61,7 +61,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { } path := filepath.Join(tempDir, "chart.yaml") - if err := os.WriteFile(path, renderedManifests.Bytes(), 0666); err != nil { + if err := os.WriteFile(path, renderedManifests.Bytes(), 0644); err != nil { return nil, fmt.Errorf("unable to write the post-render file for the helm chart") } diff --git a/src/internal/packager/kustomize/build.go b/src/internal/packager/kustomize/build.go index 3ed07ab5c2..34b182e7d4 100644 --- a/src/internal/packager/kustomize/build.go +++ b/src/internal/packager/kustomize/build.go @@ -39,5 +39,5 @@ func Build(path string, destination string, kustomizeAllowAnyDirectory bool) err return fmt.Errorf("problem converting kustomization to yaml: %w", err) } - return os.WriteFile(destination, yaml, 0666) + return os.WriteFile(destination, yaml, 0644) } diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index bf9e27534a..6958d09ed0 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -28,7 +28,7 @@ func (c *Cluster) HandleDataInjection(wg *sync.WaitGroup, data types.ZarfDataInj defer wg.Done() injectionCompletionMarker := filepath.Join(componentPath.DataInjections, config.GetDataInjectionMarker()) - if err := os.WriteFile(injectionCompletionMarker, []byte("🦄"), 0666); err != nil { + if err := os.WriteFile(injectionCompletionMarker, []byte("🦄"), 0644); err != nil { message.WarnErrf(err, "Unable to create the data injection completion marker") return } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 7c7e373afe..bf9f1eb777 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -103,7 +103,7 @@ func (o *OrasRemote) PullLayer(ctx context.Context, destinationDir string, desc rel := desc.Annotations[ocispec.AnnotationTitle] - return os.WriteFile(filepath.Join(destinationDir, rel), b, 0666) + return os.WriteFile(filepath.Join(destinationDir, rel), b, 0644) } // PullPaths pulls multiple files from the remote repository and saves them to `destinationDir`. diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index afece18228..b959441ac2 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -654,7 +654,7 @@ func (p *Packager) generatePackageChecksums() (string, error) { // Create the checksums file checksumsFilePath := p.layout.Checksums - if err := os.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n"), 0666); err != nil { + if err := os.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n"), 0644); err != nil { return "", err } diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index 3aa27d3e7c..e5f20021f0 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -150,7 +150,7 @@ func initWithoutStorageClass(t *testing.T) { storageClassFileName := "storage-class.yaml" - err = os.WriteFile(storageClassFileName, []byte(storageClassYaml), 0666) + err = os.WriteFile(storageClassFileName, []byte(storageClassYaml), 0600) require.NoError(t, err) defer e2e.CleanFiles(storageClassFileName) From 3c1003575f01a9ac6a6f4e0067e2912cc44ecac7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 18:43:05 +0000 Subject: [PATCH 067/113] refactor / delete functions --- src/pkg/oci/common.go | 7 +------ src/pkg/oci/copier.go | 4 +++- src/pkg/oci/manifest.go | 19 ------------------- src/pkg/oci/pull.go | 10 +++++++--- src/pkg/utils/helpers/slice.go | 3 +++ src/pkg/zoci/copier.go | 4 +++- 6 files changed, 17 insertions(+), 30 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index bab25b4c50..bd95d7fba7 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -183,12 +183,7 @@ func (o *OrasRemote) setRepository(ref registry.Reference) error { return err } - repo, err := remote.NewRepository(ref.String()) - if err != nil { - return err - } - - o.repo.Reference = repo.Reference + o.repo.Reference = ref o.repo.Client = client return nil diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 704068ef4a..ca7cd006de 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -11,6 +11,7 @@ import ( "io" "time" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" @@ -31,7 +32,8 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return err } - layers := srcRoot.Filter(include) + layers := helpers.Filter(srcRoot.Layers, include) + layers = append(layers, srcRoot.Config) start := time.Now() diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index 4aa5eb790f..80f2999636 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -20,11 +20,6 @@ type Manifest struct { ocispec.Manifest } -// NewManifest returns a new Manifest -func NewManifest(manifest *ocispec.Manifest) *Manifest { - return &Manifest{*manifest} -} - // Locate returns the descriptor for the first layer with the given path or digest. func (m *Manifest) Locate(pathOrDigest string) ocispec.Descriptor { return helpers.Find(m.Layers, func(layer ocispec.Descriptor) bool { @@ -37,17 +32,3 @@ func (m *Manifest) Locate(pathOrDigest string) ocispec.Descriptor { func (m *Manifest) MarshalJSON() ([]byte, error) { return json.Marshal(m.Manifest) } - -// Filter returns all the layers in the manifest satisfying `include` -func (m *Manifest) Filter(include func(d ocispec.Descriptor) bool) []ocispec.Descriptor { - if include == nil { - return append(m.Layers, m.Config) - } - var layers []ocispec.Descriptor - for _, layer := range m.Layers { - if include(layer) { - layers = append(layers, layer) - } - } - return append(layers, m.Config) -} diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index bf9f1eb777..dc692c9279 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -6,6 +6,7 @@ package oci import ( "context" + "errors" "os" "path/filepath" @@ -94,14 +95,17 @@ func (o *OrasRemote) CopyToStore(ctx context.Context, layers []ocispec.Descripto return nil } -// PullLayer pulls a layer from the remote repository and saves it to `destinationDir/annotationTitle`. -func (o *OrasRemote) PullLayer(ctx context.Context, destinationDir string, desc ocispec.Descriptor) error { +// PullPath pulls a layer from the remote repository and saves it to `destinationDir/annotationTitle`. +func (o *OrasRemote) PullPath(ctx context.Context, destinationDir string, desc ocispec.Descriptor) error { b, err := o.FetchLayer(ctx, desc) if err != nil { return err } rel := desc.Annotations[ocispec.AnnotationTitle] + if rel == "" { + return errors.New("failed to pull layer: layer is not a file") + } return os.WriteFile(filepath.Join(destinationDir, rel), b, 0644) } @@ -121,7 +125,7 @@ func (o *OrasRemote) PullPaths(ctx context.Context, destinationDir string, paths if o.FileDescriptorExists(desc, destinationDir) { continue } - err = o.PullLayer(ctx, destinationDir, desc) + err = o.PullPath(ctx, destinationDir, desc) if err != nil { return nil, err } diff --git a/src/pkg/utils/helpers/slice.go b/src/pkg/utils/helpers/slice.go index 12b662b157..93dbbd774e 100644 --- a/src/pkg/utils/helpers/slice.go +++ b/src/pkg/utils/helpers/slice.go @@ -28,6 +28,9 @@ func Reverse[T any](s []T) (r []T) { // Filter returns a new slice with only the elements that pass the test. func Filter[T any](ss []T, test func(T) bool) (r []T) { + if test == nil { + return ss + } for _, s := range ss { if test(s) { r = append(r, s) diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index a0907a0b8a..3e6b07ea50 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -22,7 +23,8 @@ func CopyPackage(ctx context.Context, src *Remote, dst *Remote, include func(d o if err != nil { return err } - layers := srcRoot.Filter(include) + layers := helpers.Filter(srcRoot.Layers, include) + layers = append(layers, srcRoot.Config) size := oci.SumDescsSize(layers) title := fmt.Sprintf("[0/%d] layers copied", len(layers)) From e814eade591904e7a675a8b44290a3c885f4a765 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 18:48:30 +0000 Subject: [PATCH 068/113] getting rid of pull layers function --- src/pkg/oci/pull.go | 19 ------------------- src/pkg/zoci/pull.go | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index dc692c9279..554a7d9e0f 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -13,7 +13,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/file" ) // FileDescriptorExists returns true if the given file exists in the given directory with the expected SHA. @@ -45,24 +44,6 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi return actual == desc.Digest.Encoded() } -// PullLayers pulls the package from the remote repository and saves it to the given path. -func (o *OrasRemote) PullLayers(ctx context.Context, destinationDir string, concurrency int, - layersToPull []ocispec.Descriptor) ([]ocispec.Descriptor, error) { - // de-duplicate layers - layersToPull = RemoveDuplicateDescriptors(layersToPull) - - dst, err := file.New(destinationDir) - if err != nil { - return nil, err - } - defer dst.Close() - - copyOpts := o.CopyOpts - copyOpts.Concurrency = concurrency - - return layersToPull, o.CopyToStore(ctx, layersToPull, dst, copyOpts) -} - // CopyToStore copies the given layers from the remote repository to the given store. func (o *OrasRemote) CopyToStore(ctx context.Context, layers []ocispec.Descriptor, store oras.Target, copyOpts oras.CopyOptions) error { shas := []string{} diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index 1c8beca3c2..662ee878e4 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content/file" ) var ( @@ -65,10 +66,19 @@ func (o *Remote) PullPackage(ctx context.Context, destinationDir string, concurr layerSize := oci.SumDescsSize(layersToPull) go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, doneSaving, "Pulling", successText) - layers, err := o.PullLayers(ctx, destinationDir, concurrency, layersToPull) + dst, err := file.New(destinationDir) + if err != nil { + return nil, err + } + defer dst.Close() + + copyOpts := o.CopyOpts + copyOpts.Concurrency = concurrency + + err = o.CopyToStore(ctx, layersToPull, dst, copyOpts) doneSaving <- err <-doneSaving - return layers, err + return layersToPull, err } // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. From 5b8ba50626a56ffce05783381f12e2a270982f85 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 7 Feb 2024 21:18:05 +0000 Subject: [PATCH 069/113] delete unused file, settle down permissions --- src/extensions/bigbang/bigbang.go | 2 +- src/internal/packager/helm/post-render.go | 2 +- src/internal/packager/kustomize/build.go | 2 +- src/pkg/cluster/data.go | 2 +- src/pkg/oci/pull.go | 2 +- src/pkg/packager/create_stages.go | 2 +- src/pkg/utils/json.go | 24 ----------------------- 7 files changed, 6 insertions(+), 30 deletions(-) delete mode 100644 src/pkg/utils/json.go diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index dca401a15b..2259050106 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -465,7 +465,7 @@ func addBigBangManifests(YOLO bool, manifestDir string, cfg *extensions.BigBang) return err } - if err := os.WriteFile(path, out, 0644); err != nil { + if err := os.WriteFile(path, out, 0600); err != nil { return err } diff --git a/src/internal/packager/helm/post-render.go b/src/internal/packager/helm/post-render.go index 2a22cf5a50..b71d14912b 100644 --- a/src/internal/packager/helm/post-render.go +++ b/src/internal/packager/helm/post-render.go @@ -61,7 +61,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { } path := filepath.Join(tempDir, "chart.yaml") - if err := os.WriteFile(path, renderedManifests.Bytes(), 0644); err != nil { + if err := os.WriteFile(path, renderedManifests.Bytes(), 0600); err != nil { return nil, fmt.Errorf("unable to write the post-render file for the helm chart") } diff --git a/src/internal/packager/kustomize/build.go b/src/internal/packager/kustomize/build.go index 34b182e7d4..cde836c2aa 100644 --- a/src/internal/packager/kustomize/build.go +++ b/src/internal/packager/kustomize/build.go @@ -39,5 +39,5 @@ func Build(path string, destination string, kustomizeAllowAnyDirectory bool) err return fmt.Errorf("problem converting kustomization to yaml: %w", err) } - return os.WriteFile(destination, yaml, 0644) + return os.WriteFile(destination, yaml, 0600) } diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index 6958d09ed0..f6b2f1bed3 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -28,7 +28,7 @@ func (c *Cluster) HandleDataInjection(wg *sync.WaitGroup, data types.ZarfDataInj defer wg.Done() injectionCompletionMarker := filepath.Join(componentPath.DataInjections, config.GetDataInjectionMarker()) - if err := os.WriteFile(injectionCompletionMarker, []byte("🦄"), 0644); err != nil { + if err := os.WriteFile(injectionCompletionMarker, []byte("🦄"), 0600); err != nil { message.WarnErrf(err, "Unable to create the data injection completion marker") return } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 554a7d9e0f..d480082ab5 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -88,7 +88,7 @@ func (o *OrasRemote) PullPath(ctx context.Context, destinationDir string, desc o return errors.New("failed to pull layer: layer is not a file") } - return os.WriteFile(filepath.Join(destinationDir, rel), b, 0644) + return os.WriteFile(filepath.Join(destinationDir, rel), b, 0600) } // PullPaths pulls multiple files from the remote repository and saves them to `destinationDir`. diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index b959441ac2..2e0e0a6e3d 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -654,7 +654,7 @@ func (p *Packager) generatePackageChecksums() (string, error) { // Create the checksums file checksumsFilePath := p.layout.Checksums - if err := os.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n"), 0644); err != nil { + if err := os.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n"), 0600); err != nil { return "", err } diff --git a/src/pkg/utils/json.go b/src/pkg/utils/json.go deleted file mode 100644 index fe14f7289d..0000000000 --- a/src/pkg/utils/json.go +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package utils provides generic utility functions. -package utils - -import ( - "encoding/json" - "os" -) - -// WriteJSON writes a given json struct to a json file on disk. -func WriteJSON(path string, v any) error { - // Remove any file that might already exist - _ = os.Remove(path) - - data, err := json.Marshal(v) - if err != nil { - return err - } - - // Create the index.json file and save the data to it - return os.WriteFile(path, data, 0644) -} From f024a9febc0996785009e69cdb662a25f9a66b40 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 8 Feb 2024 19:03:28 +0000 Subject: [PATCH 070/113] lint --- src/internal/packager/images/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 5b7f4c1c51..95187d4e6e 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -376,7 +376,7 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { }() } - onImageSavingProgress := func(finishedImage digestInfo, iteration int) { + onImageSavingProgress := func(finishedImage digestInfo, _ int) { referenceToDigest[finishedImage.refInfo.Reference] = finishedImage.digest } From 4274711f5bb551ad61bd9cc3c9d7d67322fb5a1a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 8 Feb 2024 20:54:47 +0000 Subject: [PATCH 071/113] moving progresswriter and logger to helpers --- src/pkg/oci/common.go | 50 ++--------------------------- src/pkg/oci/copier.go | 4 +-- src/pkg/utils/helpers/log.go | 51 ++++++++++++++++++++++++++++++ src/pkg/utils/helpers/transport.go | 9 ++---- 4 files changed, 58 insertions(+), 56 deletions(-) create mode 100644 src/pkg/utils/helpers/log.go diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index bd95d7fba7..2d9ca7ec0f 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -6,7 +6,6 @@ package oci import ( "fmt" - "io" "net/http" "strings" @@ -25,49 +24,6 @@ const ( MultiOS = "multi" ) -// Debug does nothing -func (l *DiscardLogger) Debug(_ string, _ ...any) {} - -// Info does nothing -func (l *DiscardLogger) Info(_ string, _ ...any) {} - -// Warn does nothing -func (l *DiscardLogger) Warn(_ string, _ ...any) {} - -// Error does nothing -func (l *DiscardLogger) Error(_ string, _ ...any) {} - -// DiscardLogger is the default if the WithLogger modifier is not used -// It discards all logs -type DiscardLogger struct{} - -// Logger is an interface built to -type Logger interface { - Debug(msg string, args ...any) - Info(msg string, args ...any) - Warn(msg string, args ...any) - Error(msg string, args ...any) -} - -// Write doesn't do anything but satisfy implementation -func (DiscardProgressWriter) Write(p []byte) (int, error) { - return len(p), nil -} - -// UpdateTitle doesn't do anything but satisfy implementation -func (DiscardProgressWriter) UpdateTitle(_ string) {} - -// DiscardProgressWriter is a ProgressWriter in which all calls succeed without doing anything -// Use this or nil or if you don't care about writing progress -type DiscardProgressWriter struct{} - -// ProgressWriter wraps io.Writer, but also includes an updateTitle function to give the user -// additional context on what's going on. Useful in OCI for tracking layers -type ProgressWriter interface { - UpdateTitle(string) - io.Writer -} - // OrasRemote is a wrapper around the Oras remote repository that includes a progress bar for interactive feedback. type OrasRemote struct { repo *remote.Repository @@ -75,7 +31,7 @@ type OrasRemote struct { Transport *helpers.Transport CopyOpts oras.CopyOptions targetPlatform *ocispec.Platform - log Logger + log helpers.Logger } // Modifier is a function that modifies an OrasRemote @@ -118,7 +74,7 @@ func WithUserAgent(userAgent string) Modifier { } // WithLogger sets the logger for the remote -func WithLogger(logger Logger) Modifier { +func WithLogger(logger helpers.Logger) Modifier { return func(o *OrasRemote) { o.log = logger } @@ -145,7 +101,7 @@ func NewOrasRemote(url string, platform ocispec.Platform, mods ...Modifier) (*Or } if o.log == nil { - o.log = &DiscardLogger{} + o.log = &helpers.DiscardLogger{} } if err := o.setRepository(ref); err != nil { diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index ca7cd006de..35a078c8a5 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -19,9 +19,9 @@ import ( // Copy copies an artifact from one OCI registry to another func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, - include func(d ocispec.Descriptor) bool, concurrency int, progressBar ProgressWriter) (err error) { + include func(d ocispec.Descriptor) bool, concurrency int, progressBar helpers.ProgressWriter) (err error) { if progressBar == nil { - progressBar = DiscardProgressWriter{} + progressBar = helpers.DiscardProgressWriter{} } // create a new semaphore to limit concurrency sem := semaphore.NewWeighted(int64(concurrency)) diff --git a/src/pkg/utils/helpers/log.go b/src/pkg/utils/helpers/log.go new file mode 100644 index 0000000000..7b4be4fef7 --- /dev/null +++ b/src/pkg/utils/helpers/log.go @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package helpers provides generic helper functions with no external imports +package helpers + +import "io" + +// Debug does nothing +func (l *DiscardLogger) Debug(_ string, _ ...any) {} + +// Info does nothing +func (l *DiscardLogger) Info(_ string, _ ...any) {} + +// Warn does nothing +func (l *DiscardLogger) Warn(_ string, _ ...any) {} + +// Error does nothing +func (l *DiscardLogger) Error(_ string, _ ...any) {} + +// DiscardLogger is the default if the WithLogger modifier is not used +// It discards all logs +type DiscardLogger struct{} + +// Write doesn't do anything but satisfy implementation +func (DiscardProgressWriter) Write(p []byte) (int, error) { + return len(p), nil +} + +// UpdateTitle doesn't do anything but satisfy implementation +func (DiscardProgressWriter) UpdateTitle(_ string) {} + +// DiscardProgressWriter is a ProgressWriter in which all calls succeed without doing anything +// Use this or nil or if you don't care about writing progress +type DiscardProgressWriter struct{} + +// Logger allows for varying levels of logging depending on importance +// Arguments are intended to be used in printf-style +type Logger interface { + Debug(msg string, args ...any) + Info(msg string, args ...any) + Warn(msg string, args ...any) + Error(msg string, args ...any) +} + +// ProgressWriter wraps io.Writer, but also includes an updateTitle function to give the user +// additional context on what's going on. Useful in OCI for tracking layers +type ProgressWriter interface { + UpdateTitle(string) + io.Writer +} diff --git a/src/pkg/utils/helpers/transport.go b/src/pkg/utils/helpers/transport.go index d6e19cc0e1..577fe77ed0 100644 --- a/src/pkg/utils/helpers/transport.go +++ b/src/pkg/utils/helpers/transport.go @@ -12,20 +12,15 @@ import ( "oras.land/oras-go/v2/registry/remote/retry" ) -type progressWriter interface { - UpdateTitle(string) - io.Writer -} - // Transport is an http.RoundTripper that keeps track of the in-flight // request and add hooks to report upload progress. type Transport struct { Base http.RoundTripper - ProgressBar progressWriter + ProgressBar ProgressWriter } // NewTransport returns a custom transport that tracks an http.RoundTripper and a message.ProgressBar. -func NewTransport(base http.RoundTripper, bar progressWriter) *Transport { +func NewTransport(base http.RoundTripper, bar ProgressWriter) *Transport { return &Transport{ Base: base, ProgressBar: bar, From 849aeffed35892280212a8ffb652327e4cceb4fb Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:58:29 -0500 Subject: [PATCH 072/113] Update src/pkg/oci/common.go Co-authored-by: razzle --- src/pkg/oci/common.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 2d9ca7ec0f..2bcb1b22de 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -94,7 +94,8 @@ func NewOrasRemote(url string, platform ocispec.Platform, mods ...Modifier) (*Or o := &OrasRemote{ repo: &remote.Repository{Client: client}, Transport: helpers.NewTransport(transport, nil), - targetPlatform: &platform} + targetPlatform: &platform, + } for _, mod := range mods { mod(o) From 85db8a474c52e1c057aacf08bedc0e533c1fab61 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:59:04 -0500 Subject: [PATCH 073/113] Update src/pkg/oci/push.go Co-authored-by: razzle --- src/pkg/oci/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 16ca2e5363..01ff9b711a 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -53,7 +53,7 @@ func (o *OrasRemote) PushManifestConfigFromMetadata(ctx context.Context, annotat if err != nil { return nil, err } - // If Media type is not set it will be set to the default + // If media type is not set it will be set to the default return o.PushLayer(ctx, manifestConfigBytes, configMediaType) } From a944089f30cbaab3932e16409148865489cd55a9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 8 Feb 2024 21:01:42 +0000 Subject: [PATCH 074/113] handle error, refactor --- src/pkg/zoci/fetch.go | 12 ++++++------ src/pkg/zoci/pull.go | 30 +++++++++++++++--------------- src/pkg/zoci/push.go | 24 +++++++++++++----------- 3 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/pkg/zoci/fetch.go b/src/pkg/zoci/fetch.go index 8d3c27d74e..f9684be74e 100644 --- a/src/pkg/zoci/fetch.go +++ b/src/pkg/zoci/fetch.go @@ -14,19 +14,19 @@ import ( ) // FetchZarfYAML fetches the zarf.yaml file from the remote repository. -func (o *Remote) FetchZarfYAML(ctx context.Context) (pkg types.ZarfPackage, err error) { - manifest, err := o.FetchRoot(ctx) +func (r *Remote) FetchZarfYAML(ctx context.Context) (pkg types.ZarfPackage, err error) { + manifest, err := r.FetchRoot(ctx) if err != nil { return pkg, err } - return oci.FetchYAMLFile[types.ZarfPackage](ctx, o.FetchLayer, manifest, layout.ZarfYAML) + return oci.FetchYAMLFile[types.ZarfPackage](ctx, r.FetchLayer, manifest, layout.ZarfYAML) } // FetchImagesIndex fetches the images/index.json file from the remote repository. -func (o *Remote) FetchImagesIndex(ctx context.Context) (index *ocispec.Index, err error) { - manifest, err := o.FetchRoot(ctx) +func (r *Remote) FetchImagesIndex(ctx context.Context) (index *ocispec.Index, err error) { + manifest, err := r.FetchRoot(ctx) if err != nil { return index, err } - return oci.FetchJSONFile[*ocispec.Index](ctx, o.FetchLayer, manifest, ZarfPackageIndexPath) + return oci.FetchJSONFile[*ocispec.Index](ctx, r.FetchLayer, manifest, ZarfPackageIndexPath) } diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index 662ee878e4..80919c2751 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -40,11 +40,11 @@ var ( // - zarf.yaml // - checksums.txt // - zarf.yaml.sig -func (o *Remote) PullPackage(ctx context.Context, destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { +func (r *Remote) PullPackage(ctx context.Context, destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { isPartialPull := len(layersToPull) > 0 - message.Debugf("Pulling %s", o.Repo().Reference) + message.Debugf("Pulling %s", r.Repo().Reference) - manifest, err := o.FetchRoot(ctx) + manifest, err := r.FetchRoot(ctx) if err != nil { return nil, err } @@ -61,7 +61,7 @@ func (o *Remote) PullPackage(ctx context.Context, destinationDir string, concurr // Create a thread to update a progress bar as we save the package to disk doneSaving := make(chan error) - successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+o.Repo().Reference.String()) + successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+r.Repo().Reference.String()) layerSize := oci.SumDescsSize(layersToPull) go utils.RenderProgressBarForLocalDirWrite(destinationDir, layerSize, doneSaving, "Pulling", successText) @@ -72,10 +72,10 @@ func (o *Remote) PullPackage(ctx context.Context, destinationDir string, concurr } defer dst.Close() - copyOpts := o.CopyOpts + copyOpts := r.CopyOpts copyOpts.Concurrency = concurrency - err = o.CopyToStore(ctx, layersToPull, dst, copyOpts) + err = r.CopyToStore(ctx, layersToPull, dst, copyOpts) doneSaving <- err <-doneSaving return layersToPull, err @@ -85,13 +85,13 @@ func (o *Remote) PullPackage(ctx context.Context, destinationDir string, concurr // It also retrieves the descriptors for all image layers that are required by the components. // // It also respects the `required` flag on components, and will retrieve all necessary layers for required components. -func (o *Remote) LayersFromRequestedComponents(ctx context.Context, requestedComponents []string) (layers []ocispec.Descriptor, err error) { - root, err := o.FetchRoot(ctx) +func (r *Remote) LayersFromRequestedComponents(ctx context.Context, requestedComponents []string) (layers []ocispec.Descriptor, err error) { + root, err := r.FetchRoot(ctx) if err != nil { return nil, err } - pkg, err := o.FetchZarfYAML(ctx) + pkg, err := r.FetchZarfYAML(ctx) if err != nil { return nil, err } @@ -124,7 +124,7 @@ func (o *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom if len(images) > 0 { // Add the image index and the oci-layout layers layers = append(layers, root.Locate(ZarfPackageIndexPath), root.Locate(ZarfPackageLayoutPath)) - index, err := o.FetchImagesIndex(ctx) + index, err := r.FetchImagesIndex(ctx) if err != nil { return nil, err } @@ -145,7 +145,7 @@ func (o *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom // even though these are technically image manifests, we store them as Zarf blobs manifestDescriptor.MediaType = ZarfLayerMediaTypeBlob - manifest, err := o.FetchManifest(ctx, manifestDescriptor) + manifest, err := r.FetchManifest(ctx, manifestDescriptor) if err != nil { return nil, err } @@ -164,11 +164,11 @@ func (o *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom } // PullPackageMetadata pulls the package metadata from the remote repository and saves it to `destinationDir`. -func (o *Remote) PullPackageMetadata(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullPaths(ctx, destinationDir, PackageAlwaysPull) +func (r *Remote) PullPackageMetadata(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { + return r.PullPaths(ctx, destinationDir, PackageAlwaysPull) } // PullPackageSBOM pulls the package's sboms.tar from the remote repository and saves it to `destinationDir`. -func (o *Remote) PullPackageSBOM(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { - return o.PullPaths(ctx, destinationDir, []string{layout.SBOMTar}) +func (r *Remote) PullPackageSBOM(ctx context.Context, destinationDir string) ([]ocispec.Descriptor, error) { + return r.PullPaths(ctx, destinationDir, []string{layout.SBOMTar}) } diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 9124274787..0d035b5181 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -19,14 +19,14 @@ import ( ) // PublishPackage publishes the zarf package to the remote repository. -func (o *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int) error { +func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, paths *layout.PackagePaths, concurrency int) error { src, err := file.New(paths.Base) if err != nil { return err } defer src.Close() - message.Infof("Publishing package to %s", o.Repo().Reference) + message.Infof("Publishing package to %s", r.Repo().Reference) spinner := message.NewProgressSpinner("") defer spinner.Stop() @@ -45,7 +45,7 @@ func (o *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat } spinner.Successf("Prepared all layers") - copyOpts := o.CopyOpts + copyOpts := r.CopyOpts copyOpts.Concurrency = concurrency total := oci.SumDescsSize(descs) @@ -53,36 +53,38 @@ func (o *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat // assumes referrers API is not supported since OCI artifact // media type is not supported - o.Repo().SetReferrersCapability(false) + r.Repo().SetReferrersCapability(false) // push the manifest config // since this config is so tiny, and the content is not used again // it is not logged to the progress, but will error if it fails - manifestConfigDesc, err := o.PushManifestConfigFromMetadata(ctx, annotations, ZarfConfigMediaType) + manifestConfigDesc, err := r.PushManifestConfigFromMetadata(ctx, annotations, ZarfConfigMediaType) if err != nil { return err } - root, err := o.GeneratePackManifest(ctx, src, descs, manifestConfigDesc, annotations) + root, err := r.GeneratePackManifest(ctx, src, descs, manifestConfigDesc, annotations) if err != nil { return err } - progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Repo().Reference.Repository, o.Repo().Reference.Reference)) - o.Transport.ProgressBar = progressBar + progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", r.Repo().Reference.Repository, r.Repo().Reference.Reference)) + r.Transport.ProgressBar = progressBar - publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), o.Repo(), "", copyOpts) + publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), r.Repo(), "", copyOpts) if err != nil { return err } - o.UpdateIndex(ctx, o.Repo().Reference.Reference, publishedDesc) + if err := r.UpdateIndex(ctx, r.Repo().Reference.Reference, publishedDesc); err != nil { + return err + } if err != nil { progressBar.Stop() return fmt.Errorf("unable to publish package: %w", err) } - progressBar.Successf("Published %s [%s]", o.Repo().Reference, ZarfLayerMediaTypeBlob) + progressBar.Successf("Published %s [%s]", r.Repo().Reference, ZarfLayerMediaTypeBlob) return nil } From 13fc297de4a965c39de9d0cdeee3639f11e0889c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 8 Feb 2024 21:24:35 +0000 Subject: [PATCH 075/113] Make copy package opinated to zarf --- src/pkg/packager/publish.go | 44 ++++--------------------------------- src/pkg/zoci/copier.go | 44 ++++++++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 45 deletions(-) diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 0c7f6cfb6b..9596f98a40 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -5,7 +5,6 @@ package packager import ( - "bytes" "context" "fmt" "os" @@ -20,7 +19,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/content" ) // Publish publishes the package to a registry @@ -32,55 +30,21 @@ func (p *Packager) Publish() (err error) { // w/o layers touching the filesystem srcRemote := p.source.(*sources.OCISource).Remote - parts := strings.Split(srcRemote.Repo().Reference.Repository, "/") - packageName := parts[len(parts)-1] - - p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName - arch := config.GetArch() - dstRemote, err := zoci.NewRemote(p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch)) - if err != nil { - return err - } - srcRoot, err := srcRemote.ResolveRoot(ctx) + dstRemote, err := zoci.NewRemote(p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch)) if err != nil { return err } - pkg, err := srcRemote.FetchZarfYAML(ctx) - if err != nil { - return err - } + parts := strings.Split(srcRemote.Repo().Reference.Repository, "/") + packageName := parts[len(parts)-1] - // ensure cli arch matches package arch - if pkg.Build.Architecture != arch { - return fmt.Errorf("architecture mismatch (specified: %q, found %q)", arch, pkg.Build.Architecture) - } + p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName if err := zoci.CopyPackage(ctx, srcRemote, dstRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { return err } - - srcManifest, err := srcRemote.FetchRoot(ctx) - if err != nil { - return err - } - b, err := srcManifest.MarshalJSON() - if err != nil { - return err - } - expected := content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, b) - - if err := dstRemote.Repo().Manifests().PushReference(ctx, expected, bytes.NewReader(b), srcRoot.Digest.String()); err != nil { - return err - } - - tag := srcRemote.Repo().Reference.Reference - if err := dstRemote.UpdateIndex(ctx, tag, expected); err != nil { - return err - } - message.Infof("Published %s to %s", srcRemote.Repo().Reference, dstRemote.Repo().Reference) return nil } diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index 3e6b07ea50..23ab547f58 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -5,26 +5,38 @@ package zoci import ( + "bytes" "context" "fmt" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" ) // CopyPackage copies a zarf package from one OCI registry to another -// TODO (@AustinAbro321) we should see if we should also copy the manifests for a complete copy in this function -// Either through ORAS or additional logic found in callers of this function func CopyPackage(ctx context.Context, src *Remote, dst *Remote, include func(d ocispec.Descriptor) bool, concurrency int) error { - srcRoot, err := src.FetchRoot(ctx) + arch := config.GetArch() + pkg, err := src.FetchZarfYAML(ctx) if err != nil { return err } - layers := helpers.Filter(srcRoot.Layers, include) - layers = append(layers, srcRoot.Config) + + // ensure cli arch matches package arch + if pkg.Build.Architecture != arch { + return fmt.Errorf("architecture mismatch (specified: %q, found %q)", arch, pkg.Build.Architecture) + } + + srcManifest, err := src.FetchRoot(ctx) + if err != nil { + return err + } + layers := helpers.Filter(srcManifest.Layers, include) + layers = append(layers, srcManifest.Config) size := oci.SumDescsSize(layers) title := fmt.Sprintf("[0/%d] layers copied", len(layers)) @@ -35,5 +47,27 @@ func CopyPackage(ctx context.Context, src *Remote, dst *Remote, include func(d o return err } progressBar.Successf("Copied %s", src.Repo().Reference) + + srcRoot, err := src.ResolveRoot(ctx) + if err != nil { + return err + } + + b, err := srcManifest.MarshalJSON() + if err != nil { + return err + } + expected := content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, b) + + if err := dst.Repo().Manifests().PushReference(ctx, expected, bytes.NewReader(b), srcRoot.Digest.String()); err != nil { + return err + } + + tag := src.Repo().Reference.Reference + if err := dst.UpdateIndex(ctx, tag, expected); err != nil { + return err + } + + message.Infof("Published %s to %s", src.Repo().Reference, dst.Repo().Reference) return nil } From c9d836944d45b28b8e9f6560faf5a04b304c781a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 8 Feb 2024 21:59:58 +0000 Subject: [PATCH 076/113] take copy opts out of oras remote --- src/pkg/oci/common.go | 14 -------------- src/pkg/oci/copier.go | 9 +++++++++ src/pkg/packager/composer/oci.go | 3 +-- src/pkg/zoci/pull.go | 2 +- src/pkg/zoci/push.go | 2 +- 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 2bcb1b22de..d3b2720b0d 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -13,7 +13,6 @@ import ( "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2" "oras.land/oras-go/v2/registry" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" @@ -29,7 +28,6 @@ type OrasRemote struct { repo *remote.Repository root *Manifest Transport *helpers.Transport - CopyOpts oras.CopyOptions targetPlatform *ocispec.Platform log helpers.Logger } @@ -37,13 +35,6 @@ type OrasRemote struct { // Modifier is a function that modifies an OrasRemote type Modifier func(*OrasRemote) -// WithCopyOpts sets the copy options for the remote -func WithCopyOpts(opts oras.CopyOptions) Modifier { - return func(o *OrasRemote) { - o.CopyOpts = opts - } -} - // WithPlainHTTP sets the plain HTTP flag for the remote func WithPlainHTTP(plainHTTP bool) Modifier { return func(o *OrasRemote) { @@ -109,11 +100,6 @@ func NewOrasRemote(url string, platform ocispec.Platform, mods ...Modifier) (*Or return nil, err } - copyOpts := oras.DefaultCopyOptions - copyOpts.OnCopySkipped = o.printLayerSkipped - copyOpts.PostCopy = o.printLayerCopied - o.CopyOpts = copyOpts - return o, nil } diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 35a078c8a5..f19f98b2d0 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -15,6 +15,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" + "oras.land/oras-go/v2" ) // Copy copies an artifact from one OCI registry to another @@ -111,3 +112,11 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return nil } + +// GetCopyOpts returns the default copy options +func (o *OrasRemote) GetCopyOpts() oras.CopyOptions { + copyOpts := oras.DefaultCopyOptions + copyOpts.OnCopySkipped = o.printLayerSkipped + copyOpts.PostCopy = o.printLayerCopied + return copyOpts +} diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 491f6affd1..d0c71feb58 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -97,11 +97,10 @@ func (ic *ImportChain) fetchOCISkeleton() error { if err != nil { return err } else if !exists { - copyOpts := remote.CopyOpts doneSaving := make(chan error) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) go utils.RenderProgressBarForLocalDirWrite(cache, componentDesc.Size, doneSaving, "Pulling", successText) - err = remote.CopyToStore(ctx, []ocispec.Descriptor{componentDesc}, store, copyOpts) + err = remote.CopyToStore(ctx, []ocispec.Descriptor{componentDesc}, store, remote.GetCopyOpts()) doneSaving <- err <-doneSaving if err != nil { diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index 80919c2751..b0d5462c80 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -72,7 +72,7 @@ func (r *Remote) PullPackage(ctx context.Context, destinationDir string, concurr } defer dst.Close() - copyOpts := r.CopyOpts + copyOpts := r.GetCopyOpts() copyOpts.Concurrency = concurrency err = r.CopyToStore(ctx, layersToPull, dst, copyOpts) diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 0d035b5181..8c64a5c125 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -45,7 +45,7 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat } spinner.Successf("Prepared all layers") - copyOpts := r.CopyOpts + copyOpts := r.GetCopyOpts() copyOpts.Concurrency = concurrency total := oci.SumDescsSize(descs) From c35481402139162fb631bd04605ef075d0593a7d Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 9 Feb 2024 14:04:39 +0000 Subject: [PATCH 077/113] rename --- src/pkg/oci/copier.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index f19f98b2d0..16cef39298 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -55,8 +55,8 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, } if exists { src.log.Debug("Layer already exists in destination, skipping") - data := make([]byte, layer.Size) - progressBar.Write(data) + b := make([]byte, layer.Size) + progressBar.Write(b) progressBar.UpdateTitle(fmt.Sprintf("[%d/%d] layers copied", idx+1, len(layers))) sem.Release(1) continue From f43b05d18171ba6c815077c464b5cfde6f58d0d4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 9 Feb 2024 15:43:06 +0000 Subject: [PATCH 078/113] adding constants for file permissions --- src/cmd/tools/helm/repo_add.go | 3 ++- src/cmd/tools/helm/repo_index.go | 5 +++-- src/cmd/tools/helm/repo_remove.go | 3 ++- src/cmd/tools/zarf.go | 11 ++++++----- src/extensions/bigbang/bigbang.go | 2 +- src/extensions/bigbang/flux.go | 2 +- src/internal/packager/helm/post-render.go | 3 ++- src/internal/packager/kustomize/build.go | 3 ++- src/pkg/cluster/data.go | 3 ++- src/pkg/oci/pull.go | 2 +- src/pkg/packager/create_stages.go | 6 +++--- src/pkg/utils/helpers/io.go | 13 +++++++++++++ src/pkg/utils/image.go | 3 ++- src/pkg/utils/io.go | 8 ++++---- src/test/e2e/00_use_cli_test.go | 3 ++- src/test/e2e/20_zarf_init_test.go | 3 ++- src/test/external/ext_out_cluster_test.go | 3 ++- 17 files changed, 50 insertions(+), 26 deletions(-) create mode 100644 src/pkg/utils/helpers/io.go diff --git a/src/cmd/tools/helm/repo_add.go b/src/cmd/tools/helm/repo_add.go index ce2e236267..436b596ed9 100644 --- a/src/cmd/tools/helm/repo_add.go +++ b/src/cmd/tools/helm/repo_add.go @@ -30,6 +30,7 @@ import ( "strings" "time" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/gofrs/flock" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -216,7 +217,7 @@ func (o *repoAddOptions) run(out io.Writer) error { f.Update(&c) - if err := f.WriteFile(o.repoFile, 0600); err != nil { + if err := f.WriteFile(o.repoFile, helpers.ReadWriteUser); err != nil { return err } fmt.Fprintf(out, "%q has been added to your repositories\n", o.name) diff --git a/src/cmd/tools/helm/repo_index.go b/src/cmd/tools/helm/repo_index.go index 79f941e79a..2da5b3fcba 100644 --- a/src/cmd/tools/helm/repo_index.go +++ b/src/cmd/tools/helm/repo_index.go @@ -26,6 +26,7 @@ import ( "os" "path/filepath" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -100,7 +101,7 @@ func index(dir, url, mergeTo string) error { var i2 *repo.IndexFile if _, err := os.Stat(mergeTo); os.IsNotExist(err) { i2 = repo.NewIndexFile() - i2.WriteFile(mergeTo, 0644) + i2.WriteFile(mergeTo, helpers.WriteUserReadAll) } else { i2, err = repo.LoadIndexFile(mergeTo) if err != nil { @@ -110,5 +111,5 @@ func index(dir, url, mergeTo string) error { i.Merge(i2) } i.SortEntries() - return i.WriteFile(out, 0644) + return i.WriteFile(out, helpers.WriteUserReadAll) } diff --git a/src/cmd/tools/helm/repo_remove.go b/src/cmd/tools/helm/repo_remove.go index 61340f8a49..594e30d2b3 100644 --- a/src/cmd/tools/helm/repo_remove.go +++ b/src/cmd/tools/helm/repo_remove.go @@ -27,6 +27,7 @@ import ( "os" "path/filepath" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -72,7 +73,7 @@ func (o *repoRemoveOptions) run(out io.Writer) error { if !r.Remove(name) { return errors.Errorf("no repo named %q found", name) } - if err := r.WriteFile(o.repoFile, 0600); err != nil { + if err := r.WriteFile(o.repoFile, helpers.ReadWriteUser); err != nil { return err } diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index f8b061c8f7..b8b5db3c96 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -21,6 +21,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/pki" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" "github.com/sigstore/cosign/v2/pkg/cosign" @@ -205,13 +206,13 @@ var generatePKICmd = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { pki := pki.GeneratePKI(args[0], subAltNames...) - if err := os.WriteFile("tls.ca", pki.CA, 0644); err != nil { + if err := os.WriteFile("tls.ca", pki.CA, helpers.WriteUserReadAll); err != nil { message.Fatalf(err, lang.ErrWritingFile, "tls.ca", err.Error()) } - if err := os.WriteFile("tls.crt", pki.Cert, 0644); err != nil { + if err := os.WriteFile("tls.crt", pki.Cert, helpers.WriteUserReadAll); err != nil { message.Fatalf(err, lang.ErrWritingFile, "tls.crt", err.Error()) } - if err := os.WriteFile("tls.key", pki.Key, 0600); err != nil { + if err := os.WriteFile("tls.key", pki.Key, helpers.ReadWriteUser); err != nil { message.Fatalf(err, lang.ErrWritingFile, "tls.key", err.Error()) } message.Successf(lang.CmdToolsGenPkiSuccess, args[0]) @@ -279,10 +280,10 @@ var generateKeyCmd = &cobra.Command{ } // Write the key file contents to disk - if err := os.WriteFile(prvKeyFileName, keyBytes.PrivateBytes, 0600); err != nil { + if err := os.WriteFile(prvKeyFileName, keyBytes.PrivateBytes, helpers.ReadWriteUser); err != nil { message.Fatalf(err, lang.ErrWritingFile, prvKeyFileName, err.Error()) } - if err := os.WriteFile(pubKeyFileName, keyBytes.PublicBytes, 0644); err != nil { + if err := os.WriteFile(pubKeyFileName, keyBytes.PublicBytes, helpers.WriteUserReadAll); err != nil { message.Fatalf(err, lang.ErrWritingFile, pubKeyFileName, err.Error()) } diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 2259050106..60c4267da7 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -465,7 +465,7 @@ func addBigBangManifests(YOLO bool, manifestDir string, cfg *extensions.BigBang) return err } - if err := os.WriteFile(path, out, 0600); err != nil { + if err := os.WriteFile(path, out, helpers.ReadWriteUser); err != nil { return err } diff --git a/src/extensions/bigbang/flux.go b/src/extensions/bigbang/flux.go index c1c2478399..8189eae591 100644 --- a/src/extensions/bigbang/flux.go +++ b/src/extensions/bigbang/flux.go @@ -62,7 +62,7 @@ func getFlux(baseDir string, cfg *extensions.BigBang) (manifest types.ZarfManife fluxKustomization.Patches = append(fluxKustomization.Patches, krustytypes.Patch{Path: absFluxPatchPath}) } - if err := utils.WriteYaml(kustomizePath, fluxKustomization, 0600); err != nil { + if err := utils.WriteYaml(kustomizePath, fluxKustomization, helpers.ReadWriteUser); err != nil { return manifest, images, fmt.Errorf("unable to write kustomization: %w", err) } diff --git a/src/internal/packager/helm/post-render.go b/src/internal/packager/helm/post-render.go index b71d14912b..6774582dcd 100644 --- a/src/internal/packager/helm/post-render.go +++ b/src/internal/packager/helm/post-render.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/releaseutil" @@ -61,7 +62,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { } path := filepath.Join(tempDir, "chart.yaml") - if err := os.WriteFile(path, renderedManifests.Bytes(), 0600); err != nil { + if err := os.WriteFile(path, renderedManifests.Bytes(), helpers.ReadWriteUser); err != nil { return nil, fmt.Errorf("unable to write the post-render file for the helm chart") } diff --git a/src/internal/packager/kustomize/build.go b/src/internal/packager/kustomize/build.go index cde836c2aa..8c045fe45c 100644 --- a/src/internal/packager/kustomize/build.go +++ b/src/internal/packager/kustomize/build.go @@ -8,6 +8,7 @@ import ( "fmt" "os" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "sigs.k8s.io/kustomize/api/krusty" krustytypes "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/filesys" @@ -39,5 +40,5 @@ func Build(path string, destination string, kustomizeAllowAnyDirectory bool) err return fmt.Errorf("problem converting kustomization to yaml: %w", err) } - return os.WriteFile(destination, yaml, 0600) + return os.WriteFile(destination, yaml, helpers.ReadWriteUser) } diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index f6b2f1bed3..e1239cdcfd 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" corev1 "k8s.io/api/core/v1" ) @@ -28,7 +29,7 @@ func (c *Cluster) HandleDataInjection(wg *sync.WaitGroup, data types.ZarfDataInj defer wg.Done() injectionCompletionMarker := filepath.Join(componentPath.DataInjections, config.GetDataInjectionMarker()) - if err := os.WriteFile(injectionCompletionMarker, []byte("🦄"), 0600); err != nil { + if err := os.WriteFile(injectionCompletionMarker, []byte("🦄"), helpers.ReadWriteUser); err != nil { message.WarnErrf(err, "Unable to create the data injection completion marker") return } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index d480082ab5..bc25d7619e 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -88,7 +88,7 @@ func (o *OrasRemote) PullPath(ctx context.Context, destinationDir string, desc o return errors.New("failed to pull layer: layer is not a file") } - return os.WriteFile(filepath.Join(destinationDir, rel), b, 0600) + return os.WriteFile(filepath.Join(destinationDir, rel), b, helpers.ReadWriteUser) } // PullPaths pulls multiple files from the remote repository and saves them to `destinationDir`. diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 2e0e0a6e3d..8461b5de1c 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -516,7 +516,7 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent) error if file.Executable || utils.IsDir(dst) { _ = os.Chmod(dst, 0700) } else { - _ = os.Chmod(dst, 0600) + _ = os.Chmod(dst, helpers.ReadWriteUser) } } @@ -654,7 +654,7 @@ func (p *Packager) generatePackageChecksums() (string, error) { // Create the checksums file checksumsFilePath := p.layout.Checksums - if err := os.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n"), 0600); err != nil { + if err := os.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n"), helpers.ReadWriteUser); err != nil { return "", err } @@ -680,7 +680,7 @@ func (p *Packager) loadDifferentialData() error { if err != nil { return err } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) + err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, helpers.ReadWriteUser) if err != nil { return err } diff --git a/src/pkg/utils/helpers/io.go b/src/pkg/utils/helpers/io.go new file mode 100644 index 0000000000..a1863831ef --- /dev/null +++ b/src/pkg/utils/helpers/io.go @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package helpers provides generic helper functions with no external imports +package helpers + +const ( + // ReadWriteUser is used for any internal file not generally consumed by the end user + // or any file with senstive data + ReadWriteUser = 0600 + // WriteUserReadAll is used for any non sensitive file intended to be consumed by the end user + WriteUserReadAll = 0644 +) diff --git a/src/pkg/utils/image.go b/src/pkg/utils/image.go index 46d67e8c88..0154953117 100644 --- a/src/pkg/utils/image.go +++ b/src/pkg/utils/image.go @@ -11,6 +11,7 @@ import ( "path/filepath" "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/layout" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -83,7 +84,7 @@ func AddImageNameAnnotation(ociPath string, referenceToDigest map[string]string) if err != nil { return err } - return os.WriteFile(indexPath, indexJSONBytes, 0600) + return os.WriteFile(indexPath, indexJSONBytes, helpers.ReadWriteUser) } // HasImageLayers checks if any layers in the v1.Image are known image layers. diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index 315a495df1..c8425da484 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -184,7 +184,7 @@ func ReplaceTextTemplate(path string, mappings map[string]*TextTemplate, depreca textFile.Close() - return os.WriteFile(path, []byte(text), 0600) + return os.WriteFile(path, []byte(text), helpers.ReadWriteUser) } @@ -352,7 +352,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { // create file path starting from part 001 path := fmt.Sprintf("%s.part001", srcFile) - chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) + chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, helpers.WriteUserReadAll) if err != nil { return err } @@ -386,7 +386,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { // create new file path = fmt.Sprintf("%s.part%03d", srcFile, len(fileNames)+1) - chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) + chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, helpers.WriteUserReadAll) if err != nil { return err } @@ -432,7 +432,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { // write header file path = fmt.Sprintf("%s.part000", srcFile) - if err := os.WriteFile(path, jsonData, 0644); err != nil { + if err := os.WriteFile(path, jsonData, helpers.WriteUserReadAll); err != nil { return fmt.Errorf("unable to write the file %s: %w", path, err) } fileNames = append(fileNames, path) diff --git a/src/test/e2e/00_use_cli_test.go b/src/test/e2e/00_use_cli_test.go index 2eaf6a6ed8..6efdd86b2d 100644 --- a/src/test/e2e/00_use_cli_test.go +++ b/src/test/e2e/00_use_cli_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/stretchr/testify/require" ) @@ -30,7 +31,7 @@ func TestUseCLI(t *testing.T) { e2e.CleanFiles(shasumTestFilePath) }) - err := os.WriteFile(shasumTestFilePath, []byte("random test data 🦄\n"), 0600) + err := os.WriteFile(shasumTestFilePath, []byte("random test data 🦄\n"), helpers.ReadWriteUser) require.NoError(t, err) stdOut, stdErr, err := e2e.Zarf("prepare", "sha256sum", shasumTestFilePath) diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index e5f20021f0..59b425df48 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -13,6 +13,7 @@ import ( "encoding/json" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" ) @@ -150,7 +151,7 @@ func initWithoutStorageClass(t *testing.T) { storageClassFileName := "storage-class.yaml" - err = os.WriteFile(storageClassFileName, []byte(storageClassYaml), 0600) + err = os.WriteFile(storageClassFileName, []byte(storageClassYaml), helpers.ReadWriteUser) require.NoError(t, err) defer e2e.CleanFiles(storageClassFileName) diff --git a/src/test/external/ext_out_cluster_test.go b/src/test/external/ext_out_cluster_test.go index 89f2d9d0aa..eb42a49676 100644 --- a/src/test/external/ext_out_cluster_test.go +++ b/src/test/external/ext_out_cluster_test.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "helm.sh/helm/v3/pkg/repo" @@ -173,7 +174,7 @@ func (suite *ExtOutClusterTestSuite) Test_2_AuthToPrivateHelmChart() { URL: chartURL, } repoFile.Add(entry) - utils.WriteYaml(repoPath, repoFile, 0600) + utils.WriteYaml(repoPath, repoFile, helpers.ReadWriteUser) err = exec.CmdWithPrint(zarfBinPath, findImageArgs...) suite.NoError(err, "Unable to find images, helm auth likely failed") From 0f844efcf9f74455dcbc574c29a37eee17f44b26 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 9 Feb 2024 15:52:30 +0000 Subject: [PATCH 079/113] naming commonly used file permission modes --- src/cmd/initialize.go | 2 +- src/internal/packager/helm/repo.go | 2 +- src/internal/packager/images/pull.go | 2 +- src/internal/packager/sbom/catalog.go | 5 +++-- src/internal/packager/sbom/tools.go | 3 ++- src/pkg/cluster/injector.go | 3 ++- src/pkg/layout/component.go | 19 ++++++++++--------- src/pkg/packager/composer/oci.go | 4 ++-- src/pkg/packager/create_stages.go | 2 +- src/pkg/packager/sources/cluster.go | 3 ++- src/pkg/packager/sources/tarball.go | 3 ++- src/pkg/utils/helpers/io.go | 9 +++++++-- src/pkg/utils/io.go | 4 ++-- src/pkg/utils/network.go | 2 +- 14 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index d01f753a60..bf567106ec 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -92,7 +92,7 @@ func findInitPackage(initPackageName string) (string, error) { // Create the cache directory if it doesn't exist if utils.InvalidPath(config.GetAbsCachePath()) { - if err := utils.CreateDirectory(config.GetAbsCachePath(), 0755); err != nil { + if err := utils.CreateDirectory(config.GetAbsCachePath(), helpers.WriteUserReadXAll); err != nil { message.Fatalf(err, lang.CmdInitErrUnableCreateCache, config.GetAbsCachePath()) } } diff --git a/src/internal/packager/helm/repo.go b/src/internal/packager/helm/repo.go index fe0423326d..e50c45c54c 100644 --- a/src/internal/packager/helm/repo.go +++ b/src/internal/packager/helm/repo.go @@ -204,7 +204,7 @@ func (h *Helm) DownloadPublishedChart(cosignKeyPath string) error { // Download the file into a temp directory since we don't control what name helm creates here temp := filepath.Join(h.chartPath, "temp") - if err = utils.CreateDirectory(temp, 0700); err != nil { + if err = utils.CreateDirectory(temp, helpers.ReadWriteXUser); err != nil { return fmt.Errorf("unable to create helm chart temp directory: %w", err) } defer os.RemoveAll(temp) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 95187d4e6e..9b9680ecf7 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -126,7 +126,7 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { } // Create the ImagePath directory - if err := utils.CreateDirectory(i.ImagesPath, 0755); err != nil { + if err := utils.CreateDirectory(i.ImagesPath, helpers.WriteUserReadXAll); err != nil { return nil, fmt.Errorf("failed to create image path %s: %w", i.ImagesPath, err) } diff --git a/src/internal/packager/sbom/catalog.go b/src/internal/packager/sbom/catalog.go index b0d83baee9..45e048e636 100755 --- a/src/internal/packager/sbom/catalog.go +++ b/src/internal/packager/sbom/catalog.go @@ -28,6 +28,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" v1 "github.com/google/go-containerregistry/pkg/v1" ) @@ -59,7 +60,7 @@ func Catalog(componentSBOMs map[string]*layout.ComponentSBOM, imageList []transf defer builder.spinner.Stop() // Ensure the sbom directory exists - _ = utils.CreateDirectory(builder.outputDir, 0700) + _ = utils.CreateDirectory(builder.outputDir, helpers.ReadWriteXUser) // Generate a list of images and files for the sbom viewer json, err := builder.generateJSONList(componentSBOMs, imageList) @@ -151,7 +152,7 @@ func (b *Builder) createImageSBOM(img v1.Image, src string) ([]byte, error) { imageCachePath := filepath.Join(b.cachePath, layout.ImagesDir) // Ensure the image cache directory exists. - if err := utils.CreateDirectory(imageCachePath, 0700); err != nil { + if err := utils.CreateDirectory(imageCachePath, helpers.ReadWriteXUser); err != nil { return nil, err } diff --git a/src/internal/packager/sbom/tools.go b/src/internal/packager/sbom/tools.go index d9ef4ef1f0..2fef41eb00 100644 --- a/src/internal/packager/sbom/tools.go +++ b/src/internal/packager/sbom/tools.go @@ -13,6 +13,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -49,7 +50,7 @@ func OutputSBOMFiles(sourceDir, outputDir, packageName string) (string, error) { return "", err } - if err := utils.CreateDirectory(packagePath, 0700); err != nil { + if err := utils.CreateDirectory(packagePath, helpers.ReadWriteXUser); err != nil { return "", err } diff --git a/src/pkg/cluster/injector.go b/src/pkg/cluster/injector.go index 6e6994add8..66d6feef9d 100644 --- a/src/pkg/cluster/injector.go +++ b/src/pkg/cluster/injector.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/google/go-containerregistry/pkg/crane" "github.com/mholt/archiver/v3" corev1 "k8s.io/api/core/v1" @@ -51,7 +52,7 @@ func (c *Cluster) StartInjectionMadness(tmpDir string, imagesDir string, injecto InjectorPayloadTarGz: filepath.Join(tmpDir, "payload.tar.gz"), } - if err := utils.CreateDirectory(tmp.SeedImagesDir, 0700); err != nil { + if err := utils.CreateDirectory(tmp.SeedImagesDir, helpers.ReadWriteXUser); err != nil { spinner.Fatalf(err, "Unable to create the seed images directory") } diff --git a/src/pkg/layout/component.go b/src/pkg/layout/component.go index 43123e65dc..7c00923dc9 100644 --- a/src/pkg/layout/component.go +++ b/src/pkg/layout/component.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" ) @@ -156,13 +157,13 @@ func (c *Components) Create(component types.ZarfComponent) (cp *ComponentPaths, } } - if err = utils.CreateDirectory(c.Base, 0700); err != nil { + if err = utils.CreateDirectory(c.Base, helpers.ReadWriteXUser); err != nil { return nil, err } base := filepath.Join(c.Base, name) - if err = utils.CreateDirectory(base, 0700); err != nil { + if err = utils.CreateDirectory(base, helpers.ReadWriteXUser); err != nil { return nil, err } @@ -171,26 +172,26 @@ func (c *Components) Create(component types.ZarfComponent) (cp *ComponentPaths, } cp.Temp = filepath.Join(base, TempDir) - if err = utils.CreateDirectory(cp.Temp, 0700); err != nil { + if err = utils.CreateDirectory(cp.Temp, helpers.ReadWriteXUser); err != nil { return nil, err } if len(component.Files) > 0 { cp.Files = filepath.Join(base, FilesDir) - if err = utils.CreateDirectory(cp.Files, 0700); err != nil { + if err = utils.CreateDirectory(cp.Files, helpers.ReadWriteXUser); err != nil { return nil, err } } if len(component.Charts) > 0 { cp.Charts = filepath.Join(base, ChartsDir) - if err = utils.CreateDirectory(cp.Charts, 0700); err != nil { + if err = utils.CreateDirectory(cp.Charts, helpers.ReadWriteXUser); err != nil { return nil, err } for _, chart := range component.Charts { cp.Values = filepath.Join(base, ValuesDir) if len(chart.ValuesFiles) > 0 { - if err = utils.CreateDirectory(cp.Values, 0700); err != nil { + if err = utils.CreateDirectory(cp.Values, helpers.ReadWriteXUser); err != nil { return nil, err } break @@ -200,21 +201,21 @@ func (c *Components) Create(component types.ZarfComponent) (cp *ComponentPaths, if len(component.Repos) > 0 { cp.Repos = filepath.Join(base, ReposDir) - if err = utils.CreateDirectory(cp.Repos, 0700); err != nil { + if err = utils.CreateDirectory(cp.Repos, helpers.ReadWriteXUser); err != nil { return nil, err } } if len(component.Manifests) > 0 { cp.Manifests = filepath.Join(base, ManifestsDir) - if err = utils.CreateDirectory(cp.Manifests, 0700); err != nil { + if err = utils.CreateDirectory(cp.Manifests, helpers.ReadWriteXUser); err != nil { return nil, err } } if len(component.DataInjections) > 0 { cp.DataInjections = filepath.Join(base, DataInjectionsDir) - if err = utils.CreateDirectory(cp.DataInjections, 0700); err != nil { + if err = utils.CreateDirectory(cp.DataInjections, helpers.ReadWriteXUser); err != nil { return nil, err } } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index d0c71feb58..c0748c34e6 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -66,7 +66,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { componentDesc := manifest.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", name))) cache := filepath.Join(config.GetAbsCachePath(), "oci") - if err := utils.CreateDirectory(cache, 0700); err != nil { + if err := utils.CreateDirectory(cache, helpers.ReadWriteXUser); err != nil { return err } @@ -110,7 +110,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { } } - if err := utils.CreateDirectory(dir, 0700); err != nil { + if err := utils.CreateDirectory(dir, helpers.ReadWriteXUser); err != nil { return err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 8461b5de1c..79b57890c8 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -514,7 +514,7 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent) error } if file.Executable || utils.IsDir(dst) { - _ = os.Chmod(dst, 0700) + _ = os.Chmod(dst, helpers.ReadWriteXUser) } else { _ = os.Chmod(dst, helpers.ReadWriteUser) } diff --git a/src/pkg/packager/sources/cluster.go b/src/pkg/packager/sources/cluster.go index 4718092474..d205807a03 100644 --- a/src/pkg/packager/sources/cluster.go +++ b/src/pkg/packager/sources/cluster.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -61,5 +62,5 @@ func (s *ClusterSource) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ dst.ZarfYAML = filepath.Join(dst.Base, layout.ZarfYAML) - return utils.WriteYaml(dst.ZarfYAML, dpkg.Data, 0755) + return utils.WriteYaml(dst.ZarfYAML, dpkg.Data, helpers.WriteUserReadXAll) } diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index e5c0e5fc6e..83ae824410 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" @@ -57,7 +58,7 @@ func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) dir := filepath.Dir(path) if dir != "." { - if err := os.MkdirAll(filepath.Join(dst.Base, dir), 0755); err != nil { + if err := os.MkdirAll(filepath.Join(dst.Base, dir), helpers.WriteUserReadXAll); err != nil { return err } } diff --git a/src/pkg/utils/helpers/io.go b/src/pkg/utils/helpers/io.go index a1863831ef..8c023244ad 100644 --- a/src/pkg/utils/helpers/io.go +++ b/src/pkg/utils/helpers/io.go @@ -5,9 +5,14 @@ package helpers const ( - // ReadWriteUser is used for any internal file not generally consumed by the end user - // or any file with senstive data + // ReadWriteUser is used for any internal file not normallly used by the end user or containing senstive data ReadWriteUser = 0600 // WriteUserReadAll is used for any non sensitive file intended to be consumed by the end user WriteUserReadAll = 0644 + + // ReadWriteXUser is used for any directory or executable not normally used by the end user or containing sensitive data + ReadWriteXUser = 0700 + + // WriteUserReadXAll is used for any non sensitive directory or executable intended to be consumed by the end user + WriteUserReadXAll = 0755 ) diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index c8425da484..3e0adc1a48 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -43,7 +43,7 @@ type TextTemplate struct { // MakeTempDir creates a temp directory with the zarf- prefix. func MakeTempDir(basePath string) (string, error) { if basePath != "" { - if err := CreateDirectory(basePath, 0700); err != nil { + if err := CreateDirectory(basePath, helpers.ReadWriteXUser); err != nil { return "", err } } @@ -225,7 +225,7 @@ func RecursiveFileList(dir string, pattern *regexp.Regexp, skipHidden bool) (fil // CreateParentDirectory creates the parent directory for the given file path. func CreateParentDirectory(destination string) error { parentDest := filepath.Dir(destination) - return CreateDirectory(parentDest, 0700) + return CreateDirectory(parentDest, helpers.ReadWriteXUser) } // CreatePathAndCopy creates the parent directory for the given file path and copies the source file to the destination. diff --git a/src/pkg/utils/network.go b/src/pkg/utils/network.go index 0898bda028..a4446d4d70 100644 --- a/src/pkg/utils/network.go +++ b/src/pkg/utils/network.go @@ -48,7 +48,7 @@ func DownloadToFile(src string, dst string, cosignKeyPath string) (err error) { return err } - err = CreateDirectory(filepath.Dir(dst), 0700) + err = CreateDirectory(filepath.Dir(dst), helpers.ReadWriteXUser) if err != nil { return fmt.Errorf(lang.ErrCreatingDir, filepath.Dir(dst), err.Error()) } From dc1eef9daaa293de1b6108c45e5d3f43a21eddfc Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 9 Feb 2024 18:40:49 +0000 Subject: [PATCH 080/113] rename --- src/pkg/oci/pull.go | 6 +++--- src/pkg/packager/composer/oci.go | 2 +- src/pkg/zoci/pull.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index bc25d7619e..2a1fe8f448 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -44,8 +44,8 @@ func (o *OrasRemote) FileDescriptorExists(desc ocispec.Descriptor, destinationDi return actual == desc.Digest.Encoded() } -// CopyToStore copies the given layers from the remote repository to the given store. -func (o *OrasRemote) CopyToStore(ctx context.Context, layers []ocispec.Descriptor, store oras.Target, copyOpts oras.CopyOptions) error { +// CopyToTarget copies the given layers from the remote repository to the given target +func (o *OrasRemote) CopyToTarget(ctx context.Context, layers []ocispec.Descriptor, target oras.Target, copyOpts oras.CopyOptions) error { shas := []string{} for _, layer := range layers { if len(layer.Digest.String()) > 0 { @@ -68,7 +68,7 @@ func (o *OrasRemote) CopyToStore(ctx context.Context, layers []ocispec.Descripto return oras.SkipNode } - _, err := oras.Copy(ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) + _, err := oras.Copy(ctx, o.repo, o.repo.Reference.String(), target, o.repo.Reference.String(), copyOpts) if err != nil { return err } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index c0748c34e6..7a03a9429e 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -100,7 +100,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { doneSaving := make(chan error) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) go utils.RenderProgressBarForLocalDirWrite(cache, componentDesc.Size, doneSaving, "Pulling", successText) - err = remote.CopyToStore(ctx, []ocispec.Descriptor{componentDesc}, store, remote.GetCopyOpts()) + err = remote.CopyToTarget(ctx, []ocispec.Descriptor{componentDesc}, store, remote.GetCopyOpts()) doneSaving <- err <-doneSaving if err != nil { diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index b0d5462c80..fb8cf7cae9 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -75,7 +75,7 @@ func (r *Remote) PullPackage(ctx context.Context, destinationDir string, concurr copyOpts := r.GetCopyOpts() copyOpts.Concurrency = concurrency - err = r.CopyToStore(ctx, layersToPull, dst, copyOpts) + err = r.CopyToTarget(ctx, layersToPull, dst, copyOpts) doneSaving <- err <-doneSaving return layersToPull, err From 2149a3782325f198dfdbf21b63b046ddd1128d70 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 9 Feb 2024 18:42:53 +0000 Subject: [PATCH 081/113] moving and renaming getcopyopts --- src/pkg/oci/copier.go | 9 --------- src/pkg/oci/utils.go | 9 +++++++++ src/pkg/packager/composer/oci.go | 2 +- src/pkg/zoci/pull.go | 2 +- src/pkg/zoci/push.go | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index 16cef39298..e9ea0e099e 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -15,7 +15,6 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" - "oras.land/oras-go/v2" ) // Copy copies an artifact from one OCI registry to another @@ -112,11 +111,3 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, return nil } - -// GetCopyOpts returns the default copy options -func (o *OrasRemote) GetCopyOpts() oras.CopyOptions { - copyOpts := oras.DefaultCopyOptions - copyOpts.OnCopySkipped = o.printLayerSkipped - copyOpts.PostCopy = o.printLayerCopied - return copyOpts -} diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index 42469e3c07..417ee26796 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" "oras.land/oras-go/v2/registry" ) @@ -52,3 +53,11 @@ func SumDescsSize(descs []ocispec.Descriptor) int64 { } return sum } + +// GetDefaultCopyOpts returns the default copy options +func (o *OrasRemote) GetDefaultCopyOpts() oras.CopyOptions { + copyOpts := oras.DefaultCopyOptions + copyOpts.OnCopySkipped = o.printLayerSkipped + copyOpts.PostCopy = o.printLayerCopied + return copyOpts +} diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 7a03a9429e..7046375654 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -100,7 +100,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { doneSaving := make(chan error) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+remote.Repo().Reference.String()) go utils.RenderProgressBarForLocalDirWrite(cache, componentDesc.Size, doneSaving, "Pulling", successText) - err = remote.CopyToTarget(ctx, []ocispec.Descriptor{componentDesc}, store, remote.GetCopyOpts()) + err = remote.CopyToTarget(ctx, []ocispec.Descriptor{componentDesc}, store, remote.GetDefaultCopyOpts()) doneSaving <- err <-doneSaving if err != nil { diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index fb8cf7cae9..6e357fc720 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -72,7 +72,7 @@ func (r *Remote) PullPackage(ctx context.Context, destinationDir string, concurr } defer dst.Close() - copyOpts := r.GetCopyOpts() + copyOpts := r.GetDefaultCopyOpts() copyOpts.Concurrency = concurrency err = r.CopyToTarget(ctx, layersToPull, dst, copyOpts) diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 8c64a5c125..617922b92e 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -45,7 +45,7 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat } spinner.Successf("Prepared all layers") - copyOpts := r.GetCopyOpts() + copyOpts := r.GetDefaultCopyOpts() copyOpts.Concurrency = concurrency total := oci.SumDescsSize(descs) From b2c1036c6eb8a045aaee4ff927fbf965a12ccc01 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 9 Feb 2024 19:32:21 +0000 Subject: [PATCH 082/113] Remove include as an option to zoci copy package, fix package destination --- src/pkg/packager/publish.go | 12 ++++++------ src/pkg/zoci/copier.go | 8 +++----- src/test/e2e/50_oci_publish_deploy_test.go | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 9596f98a40..21c8d9c270 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -30,6 +30,11 @@ func (p *Packager) Publish() (err error) { // w/o layers touching the filesystem srcRemote := p.source.(*sources.OCISource).Remote + parts := strings.Split(srcRemote.Repo().Reference.Repository, "/") + packageName := parts[len(parts)-1] + + p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName + arch := config.GetArch() dstRemote, err := zoci.NewRemote(p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch)) @@ -37,12 +42,7 @@ func (p *Packager) Publish() (err error) { return err } - parts := strings.Split(srcRemote.Repo().Reference.Repository, "/") - packageName := parts[len(parts)-1] - - p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName - - if err := zoci.CopyPackage(ctx, srcRemote, dstRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { + if err := zoci.CopyPackage(ctx, srcRemote, dstRemote, config.CommonOptions.OCIConcurrency); err != nil { return err } return nil diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index 23ab547f58..10d1e7b733 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -12,13 +12,12 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/content" ) // CopyPackage copies a zarf package from one OCI registry to another -func CopyPackage(ctx context.Context, src *Remote, dst *Remote, include func(d ocispec.Descriptor) bool, concurrency int) error { +func CopyPackage(ctx context.Context, src *Remote, dst *Remote, concurrency int) error { arch := config.GetArch() pkg, err := src.FetchZarfYAML(ctx) @@ -35,15 +34,14 @@ func CopyPackage(ctx context.Context, src *Remote, dst *Remote, include func(d o if err != nil { return err } - layers := helpers.Filter(srcManifest.Layers, include) - layers = append(layers, srcManifest.Config) + layers := append(srcManifest.Layers, srcManifest.Config) size := oci.SumDescsSize(layers) title := fmt.Sprintf("[0/%d] layers copied", len(layers)) progressBar := message.NewProgressBar(size, title) defer progressBar.Stop() - if err := oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, include, concurrency, progressBar); err != nil { + if err := oci.Copy(ctx, src.OrasRemote, dst.OrasRemote, nil, concurrency, progressBar); err != nil { return err } progressBar.Successf("Copied %s", src.Repo().Reference) diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 2e44beffcf..8606550c7a 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -150,7 +150,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { } require.Less(t, attempt, 5, "failed to ping registry") - err = zoci.CopyPackage(ctx, src, dst, nil, 5) + err = zoci.CopyPackage(ctx, src, dst, 5) suite.NoError(err) srcRoot, err := src.FetchRoot(ctx) From 1a5653cee7f2290ff165c4107c6b76be93f66283 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 9 Feb 2024 21:12:30 +0000 Subject: [PATCH 083/113] remove unncessary error check --- src/pkg/zoci/copier.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index 10d1e7b733..0586dc3395 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -9,7 +9,6 @@ import ( "context" "fmt" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -19,17 +18,6 @@ import ( // CopyPackage copies a zarf package from one OCI registry to another func CopyPackage(ctx context.Context, src *Remote, dst *Remote, concurrency int) error { - arch := config.GetArch() - pkg, err := src.FetchZarfYAML(ctx) - if err != nil { - return err - } - - // ensure cli arch matches package arch - if pkg.Build.Architecture != arch { - return fmt.Errorf("architecture mismatch (specified: %q, found %q)", arch, pkg.Build.Architecture) - } - srcManifest, err := src.FetchRoot(ctx) if err != nil { return err From e0e72b531b29e160867822604c37545e32ea5b31 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 12 Feb 2024 19:34:08 +0000 Subject: [PATCH 084/113] change logger to follow slog interface --- src/pkg/message/handler.go | 47 ++++++++++++++++++++++++++++++++++++++ src/pkg/message/message.go | 23 ------------------- src/pkg/oci/common.go | 12 ++++------ src/pkg/oci/copier.go | 7 +++--- src/pkg/oci/progress.go | 2 +- src/pkg/zoci/common.go | 5 +++- 6 files changed, 61 insertions(+), 35 deletions(-) create mode 100644 src/pkg/message/handler.go diff --git a/src/pkg/message/handler.go b/src/pkg/message/handler.go new file mode 100644 index 0000000000..a2269cbf2e --- /dev/null +++ b/src/pkg/message/handler.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package message provides a rich set of functions for displaying messages to the user. +package message + +import ( + "context" + "log/slog" +) + +// ZarfHandler is a simple handler that implements the slog.Handler interface +type ZarfHandler struct{} + +// Enabled is always set to true as zarf logging functions are already aware of if they are allowed to be called +func (z ZarfHandler) Enabled(_ context.Context, _ slog.Level) bool { + return true +} + +// WithAttrs is not suppported +func (z ZarfHandler) WithAttrs(_ []slog.Attr) slog.Handler { + return z +} + +// WithGroup is not supported +func (z ZarfHandler) WithGroup(_ string) slog.Handler { + return z +} + +// Handle prints the respective logging function in zarf +// This function ignores any key pairs passed through the record +func (z ZarfHandler) Handle(ctx context.Context, record slog.Record) error { + level := record.Level + message := record.Message + + switch level { + case slog.LevelDebug: + Debug(message) + case slog.LevelInfo: + Info(message) + case slog.LevelWarn: + Warn(message) + case slog.LevelError: + Warn(message) + } + return nil +} diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index 83cf9cc874..d17baca298 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -38,29 +38,6 @@ const ( TermWidth = 100 ) -// Logger is an object with access to the main printing functions in message -type Logger struct{} - -// Debug prints a debug message. -func (l *Logger) Debug(msg string, args ...any) { - Debugf(msg, args...) -} - -// Info prints an Info message. -func (l *Logger) Info(msg string, args ...any) { - Infof(msg, args...) -} - -// Warn prints a warning message -func (l *Logger) Warn(msg string, args ...any) { - Warnf(msg, args...) -} - -// Error prints a warning message -func (l *Logger) Error(msg string, args ...any) { - Warnf(msg, args...) -} - // NoProgress tracks whether spinner/progress bars show updates. var NoProgress bool diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index d3b2720b0d..495384cb42 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -6,6 +6,7 @@ package oci import ( "fmt" + "log/slog" "net/http" "strings" @@ -29,7 +30,7 @@ type OrasRemote struct { root *Manifest Transport *helpers.Transport targetPlatform *ocispec.Platform - log helpers.Logger + log slog.Logger } // Modifier is a function that modifies an OrasRemote @@ -65,7 +66,7 @@ func WithUserAgent(userAgent string) Modifier { } // WithLogger sets the logger for the remote -func WithLogger(logger helpers.Logger) Modifier { +func WithLogger(logger slog.Logger) Modifier { return func(o *OrasRemote) { o.log = logger } @@ -86,16 +87,13 @@ func NewOrasRemote(url string, platform ocispec.Platform, mods ...Modifier) (*Or repo: &remote.Repository{Client: client}, Transport: helpers.NewTransport(transport, nil), targetPlatform: &platform, + log: *slog.Default(), } for _, mod := range mods { mod(o) } - if o.log == nil { - o.log = &helpers.DiscardLogger{} - } - if err := o.setRepository(ref); err != nil { return nil, err } @@ -140,7 +138,7 @@ func (o *OrasRemote) setRepository(ref registry.Reference) error { func (o *OrasRemote) createAuthClient(ref registry.Reference) (*auth.Client, error) { client := o.repo.Client.(*auth.Client) - o.log.Debug("Loading docker config file from default config location: %s for %s", config.Dir(), ref) + o.log.Debug(fmt.Sprintf("Loading docker config file from default config location: %s for %s", config.Dir(), ref)) cfg, err := config.Load(config.Dir()) if err != nil { return nil, err diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index e9ea0e099e..e74eb9fd0a 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -40,9 +40,9 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, for idx, layer := range layers { b, err := json.MarshalIndent(layer, "", " ") if err != nil { - src.log.Debug("ERROR marshalling json: %s", err.Error()) + src.log.Debug(fmt.Sprintf("ERROR marshalling json: %s", err.Error())) } - src.log.Debug("Copying layer:", string(b)) + src.log.Debug(fmt.Sprintf("Copying layer: %s", string(b))) if err := sem.Acquire(ctx, 1); err != nil { return err } @@ -107,7 +107,8 @@ func Copy(ctx context.Context, src *OrasRemote, dst *OrasRemote, } duration := time.Since(start) - src.log.Debug("Copied", src.repo.Reference, "to", dst.repo.Reference, "with a concurrency of", concurrency, "and took", duration) + src.log.Debug(fmt.Sprintf("Copied %s to %s with a concurrency of %d and took %s", + src.repo.Reference, dst.repo.Reference, concurrency, duration)) return nil } diff --git a/src/pkg/oci/progress.go b/src/pkg/oci/progress.go index 79d383ed11..e2754bc60c 100644 --- a/src/pkg/oci/progress.go +++ b/src/pkg/oci/progress.go @@ -31,6 +31,6 @@ func (o *OrasRemote) printLayer(desc ocispec.Descriptor, suffix string) error { } else { layerInfo = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) } - o.log.Debug("%s (%s)", layerInfo, suffix) + o.log.Debug(fmt.Sprintf("%s (%s)", layerInfo, suffix)) return nil } diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index d5654ccedf..da7bb9170f 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -5,6 +5,8 @@ package zoci import ( + "log/slog" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" @@ -28,10 +30,11 @@ type Remote struct { // NewRemote returns an oras remote repository client and context for the given url // with zarf opination embedded func NewRemote(url string, platform ocispec.Platform, mods ...oci.Modifier) (*Remote, error) { + logger := *slog.New(message.ZarfHandler{}) modifiers := append([]oci.Modifier{ oci.WithPlainHTTP(config.CommonOptions.Insecure), oci.WithInsecureSkipVerify(config.CommonOptions.Insecure), - oci.WithLogger(&message.Logger{}), + oci.WithLogger(logger), oci.WithUserAgent("zarf/" + config.CLIVersion), }, mods...) remote, err := oci.NewOrasRemote(url, platform, modifiers...) From fe17afebc2ebdb029031c06627502339c9e4517e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 12 Feb 2024 20:47:49 +0000 Subject: [PATCH 085/113] rename io permissions --- src/cmd/initialize.go | 2 +- src/cmd/tools/helm/repo_index.go | 4 ++-- src/cmd/tools/zarf.go | 6 +++--- src/internal/packager/helm/repo.go | 2 +- src/internal/packager/images/pull.go | 2 +- src/internal/packager/sbom/catalog.go | 4 ++-- src/internal/packager/sbom/tools.go | 2 +- src/pkg/cluster/injector.go | 2 +- src/pkg/layout/component.go | 18 +++++++++--------- src/pkg/packager/composer/oci.go | 4 ++-- src/pkg/packager/create_stages.go | 2 +- src/pkg/packager/sources/cluster.go | 2 +- src/pkg/packager/sources/tarball.go | 2 +- src/pkg/utils/helpers/io.go | 12 ++++++------ src/pkg/utils/io.go | 10 +++++----- src/pkg/utils/network.go | 2 +- 16 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index bf567106ec..79452967db 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -92,7 +92,7 @@ func findInitPackage(initPackageName string) (string, error) { // Create the cache directory if it doesn't exist if utils.InvalidPath(config.GetAbsCachePath()) { - if err := utils.CreateDirectory(config.GetAbsCachePath(), helpers.WriteUserReadXAll); err != nil { + if err := utils.CreateDirectory(config.GetAbsCachePath(), helpers.ReadExecuteAllWriteUser); err != nil { message.Fatalf(err, lang.CmdInitErrUnableCreateCache, config.GetAbsCachePath()) } } diff --git a/src/cmd/tools/helm/repo_index.go b/src/cmd/tools/helm/repo_index.go index 2da5b3fcba..8989f10448 100644 --- a/src/cmd/tools/helm/repo_index.go +++ b/src/cmd/tools/helm/repo_index.go @@ -101,7 +101,7 @@ func index(dir, url, mergeTo string) error { var i2 *repo.IndexFile if _, err := os.Stat(mergeTo); os.IsNotExist(err) { i2 = repo.NewIndexFile() - i2.WriteFile(mergeTo, helpers.WriteUserReadAll) + i2.WriteFile(mergeTo, helpers.ReadAllWriteUser) } else { i2, err = repo.LoadIndexFile(mergeTo) if err != nil { @@ -111,5 +111,5 @@ func index(dir, url, mergeTo string) error { i.Merge(i2) } i.SortEntries() - return i.WriteFile(out, helpers.WriteUserReadAll) + return i.WriteFile(out, helpers.ReadAllWriteUser) } diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index b8b5db3c96..f4b7574f3e 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -206,10 +206,10 @@ var generatePKICmd = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { pki := pki.GeneratePKI(args[0], subAltNames...) - if err := os.WriteFile("tls.ca", pki.CA, helpers.WriteUserReadAll); err != nil { + if err := os.WriteFile("tls.ca", pki.CA, helpers.ReadAllWriteUser); err != nil { message.Fatalf(err, lang.ErrWritingFile, "tls.ca", err.Error()) } - if err := os.WriteFile("tls.crt", pki.Cert, helpers.WriteUserReadAll); err != nil { + if err := os.WriteFile("tls.crt", pki.Cert, helpers.ReadAllWriteUser); err != nil { message.Fatalf(err, lang.ErrWritingFile, "tls.crt", err.Error()) } if err := os.WriteFile("tls.key", pki.Key, helpers.ReadWriteUser); err != nil { @@ -283,7 +283,7 @@ var generateKeyCmd = &cobra.Command{ if err := os.WriteFile(prvKeyFileName, keyBytes.PrivateBytes, helpers.ReadWriteUser); err != nil { message.Fatalf(err, lang.ErrWritingFile, prvKeyFileName, err.Error()) } - if err := os.WriteFile(pubKeyFileName, keyBytes.PublicBytes, helpers.WriteUserReadAll); err != nil { + if err := os.WriteFile(pubKeyFileName, keyBytes.PublicBytes, helpers.ReadAllWriteUser); err != nil { message.Fatalf(err, lang.ErrWritingFile, pubKeyFileName, err.Error()) } diff --git a/src/internal/packager/helm/repo.go b/src/internal/packager/helm/repo.go index e50c45c54c..ce57641c91 100644 --- a/src/internal/packager/helm/repo.go +++ b/src/internal/packager/helm/repo.go @@ -204,7 +204,7 @@ func (h *Helm) DownloadPublishedChart(cosignKeyPath string) error { // Download the file into a temp directory since we don't control what name helm creates here temp := filepath.Join(h.chartPath, "temp") - if err = utils.CreateDirectory(temp, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(temp, helpers.ReadWriteExecuteUser); err != nil { return fmt.Errorf("unable to create helm chart temp directory: %w", err) } defer os.RemoveAll(temp) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 9b9680ecf7..bea3ec8abf 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -126,7 +126,7 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { } // Create the ImagePath directory - if err := utils.CreateDirectory(i.ImagesPath, helpers.WriteUserReadXAll); err != nil { + if err := utils.CreateDirectory(i.ImagesPath, helpers.ReadExecuteAllWriteUser); err != nil { return nil, fmt.Errorf("failed to create image path %s: %w", i.ImagesPath, err) } diff --git a/src/internal/packager/sbom/catalog.go b/src/internal/packager/sbom/catalog.go index 45e048e636..646f066a34 100755 --- a/src/internal/packager/sbom/catalog.go +++ b/src/internal/packager/sbom/catalog.go @@ -60,7 +60,7 @@ func Catalog(componentSBOMs map[string]*layout.ComponentSBOM, imageList []transf defer builder.spinner.Stop() // Ensure the sbom directory exists - _ = utils.CreateDirectory(builder.outputDir, helpers.ReadWriteXUser) + _ = utils.CreateDirectory(builder.outputDir, helpers.ReadWriteExecuteUser) // Generate a list of images and files for the sbom viewer json, err := builder.generateJSONList(componentSBOMs, imageList) @@ -152,7 +152,7 @@ func (b *Builder) createImageSBOM(img v1.Image, src string) ([]byte, error) { imageCachePath := filepath.Join(b.cachePath, layout.ImagesDir) // Ensure the image cache directory exists. - if err := utils.CreateDirectory(imageCachePath, helpers.ReadWriteXUser); err != nil { + if err := utils.CreateDirectory(imageCachePath, helpers.ReadWriteExecuteUser); err != nil { return nil, err } diff --git a/src/internal/packager/sbom/tools.go b/src/internal/packager/sbom/tools.go index 2fef41eb00..e964f85ece 100644 --- a/src/internal/packager/sbom/tools.go +++ b/src/internal/packager/sbom/tools.go @@ -50,7 +50,7 @@ func OutputSBOMFiles(sourceDir, outputDir, packageName string) (string, error) { return "", err } - if err := utils.CreateDirectory(packagePath, helpers.ReadWriteXUser); err != nil { + if err := utils.CreateDirectory(packagePath, helpers.ReadWriteExecuteUser); err != nil { return "", err } diff --git a/src/pkg/cluster/injector.go b/src/pkg/cluster/injector.go index 66d6feef9d..b4c465ede3 100644 --- a/src/pkg/cluster/injector.go +++ b/src/pkg/cluster/injector.go @@ -52,7 +52,7 @@ func (c *Cluster) StartInjectionMadness(tmpDir string, imagesDir string, injecto InjectorPayloadTarGz: filepath.Join(tmpDir, "payload.tar.gz"), } - if err := utils.CreateDirectory(tmp.SeedImagesDir, helpers.ReadWriteXUser); err != nil { + if err := utils.CreateDirectory(tmp.SeedImagesDir, helpers.ReadWriteExecuteUser); err != nil { spinner.Fatalf(err, "Unable to create the seed images directory") } diff --git a/src/pkg/layout/component.go b/src/pkg/layout/component.go index 7c00923dc9..6a97a9dd47 100644 --- a/src/pkg/layout/component.go +++ b/src/pkg/layout/component.go @@ -157,13 +157,13 @@ func (c *Components) Create(component types.ZarfComponent) (cp *ComponentPaths, } } - if err = utils.CreateDirectory(c.Base, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(c.Base, helpers.ReadWriteExecuteUser); err != nil { return nil, err } base := filepath.Join(c.Base, name) - if err = utils.CreateDirectory(base, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(base, helpers.ReadWriteExecuteUser); err != nil { return nil, err } @@ -172,26 +172,26 @@ func (c *Components) Create(component types.ZarfComponent) (cp *ComponentPaths, } cp.Temp = filepath.Join(base, TempDir) - if err = utils.CreateDirectory(cp.Temp, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(cp.Temp, helpers.ReadWriteExecuteUser); err != nil { return nil, err } if len(component.Files) > 0 { cp.Files = filepath.Join(base, FilesDir) - if err = utils.CreateDirectory(cp.Files, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(cp.Files, helpers.ReadWriteExecuteUser); err != nil { return nil, err } } if len(component.Charts) > 0 { cp.Charts = filepath.Join(base, ChartsDir) - if err = utils.CreateDirectory(cp.Charts, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(cp.Charts, helpers.ReadWriteExecuteUser); err != nil { return nil, err } for _, chart := range component.Charts { cp.Values = filepath.Join(base, ValuesDir) if len(chart.ValuesFiles) > 0 { - if err = utils.CreateDirectory(cp.Values, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(cp.Values, helpers.ReadWriteExecuteUser); err != nil { return nil, err } break @@ -201,21 +201,21 @@ func (c *Components) Create(component types.ZarfComponent) (cp *ComponentPaths, if len(component.Repos) > 0 { cp.Repos = filepath.Join(base, ReposDir) - if err = utils.CreateDirectory(cp.Repos, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(cp.Repos, helpers.ReadWriteExecuteUser); err != nil { return nil, err } } if len(component.Manifests) > 0 { cp.Manifests = filepath.Join(base, ManifestsDir) - if err = utils.CreateDirectory(cp.Manifests, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(cp.Manifests, helpers.ReadWriteExecuteUser); err != nil { return nil, err } } if len(component.DataInjections) > 0 { cp.DataInjections = filepath.Join(base, DataInjectionsDir) - if err = utils.CreateDirectory(cp.DataInjections, helpers.ReadWriteXUser); err != nil { + if err = utils.CreateDirectory(cp.DataInjections, helpers.ReadWriteExecuteUser); err != nil { return nil, err } } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 7046375654..9d0485ce52 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -66,7 +66,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { componentDesc := manifest.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", name))) cache := filepath.Join(config.GetAbsCachePath(), "oci") - if err := utils.CreateDirectory(cache, helpers.ReadWriteXUser); err != nil { + if err := utils.CreateDirectory(cache, helpers.ReadWriteExecuteUser); err != nil { return err } @@ -110,7 +110,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { } } - if err := utils.CreateDirectory(dir, helpers.ReadWriteXUser); err != nil { + if err := utils.CreateDirectory(dir, helpers.ReadWriteExecuteUser); err != nil { return err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 79b57890c8..5ea66cc3fb 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -514,7 +514,7 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent) error } if file.Executable || utils.IsDir(dst) { - _ = os.Chmod(dst, helpers.ReadWriteXUser) + _ = os.Chmod(dst, helpers.ReadWriteExecuteUser) } else { _ = os.Chmod(dst, helpers.ReadWriteUser) } diff --git a/src/pkg/packager/sources/cluster.go b/src/pkg/packager/sources/cluster.go index d205807a03..358a764f77 100644 --- a/src/pkg/packager/sources/cluster.go +++ b/src/pkg/packager/sources/cluster.go @@ -62,5 +62,5 @@ func (s *ClusterSource) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ dst.ZarfYAML = filepath.Join(dst.Base, layout.ZarfYAML) - return utils.WriteYaml(dst.ZarfYAML, dpkg.Data, helpers.WriteUserReadXAll) + return utils.WriteYaml(dst.ZarfYAML, dpkg.Data, helpers.ReadExecuteAllWriteUser) } diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index 83ae824410..1903e09120 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -58,7 +58,7 @@ func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) dir := filepath.Dir(path) if dir != "." { - if err := os.MkdirAll(filepath.Join(dst.Base, dir), helpers.WriteUserReadXAll); err != nil { + if err := os.MkdirAll(filepath.Join(dst.Base, dir), helpers.ReadExecuteAllWriteUser); err != nil { return err } } diff --git a/src/pkg/utils/helpers/io.go b/src/pkg/utils/helpers/io.go index 8c023244ad..726ff94fa5 100644 --- a/src/pkg/utils/helpers/io.go +++ b/src/pkg/utils/helpers/io.go @@ -7,12 +7,12 @@ package helpers const ( // ReadWriteUser is used for any internal file not normallly used by the end user or containing senstive data ReadWriteUser = 0600 - // WriteUserReadAll is used for any non sensitive file intended to be consumed by the end user - WriteUserReadAll = 0644 + // ReadAllWriteUser is used for any non sensitive file intended to be consumed by the end user + ReadAllWriteUser = 0644 - // ReadWriteXUser is used for any directory or executable not normally used by the end user or containing sensitive data - ReadWriteXUser = 0700 + // ReadWriteExecuteUser is used for any directory or executable not normally used by the end user or containing sensitive data + ReadWriteExecuteUser = 0700 - // WriteUserReadXAll is used for any non sensitive directory or executable intended to be consumed by the end user - WriteUserReadXAll = 0755 + // ReadExecuteAllWriteUser is used for any non sensitive directory or executable intended to be consumed by the end user + ReadExecuteAllWriteUser = 0755 ) diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index 3e0adc1a48..466bb912fb 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -43,7 +43,7 @@ type TextTemplate struct { // MakeTempDir creates a temp directory with the zarf- prefix. func MakeTempDir(basePath string) (string, error) { if basePath != "" { - if err := CreateDirectory(basePath, helpers.ReadWriteXUser); err != nil { + if err := CreateDirectory(basePath, helpers.ReadWriteExecuteUser); err != nil { return "", err } } @@ -225,7 +225,7 @@ func RecursiveFileList(dir string, pattern *regexp.Regexp, skipHidden bool) (fil // CreateParentDirectory creates the parent directory for the given file path. func CreateParentDirectory(destination string) error { parentDest := filepath.Dir(destination) - return CreateDirectory(parentDest, helpers.ReadWriteXUser) + return CreateDirectory(parentDest, helpers.ReadWriteExecuteUser) } // CreatePathAndCopy creates the parent directory for the given file path and copies the source file to the destination. @@ -352,7 +352,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { // create file path starting from part 001 path := fmt.Sprintf("%s.part001", srcFile) - chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, helpers.WriteUserReadAll) + chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, helpers.ReadAllWriteUser) if err != nil { return err } @@ -386,7 +386,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { // create new file path = fmt.Sprintf("%s.part%03d", srcFile, len(fileNames)+1) - chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, helpers.WriteUserReadAll) + chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, helpers.ReadAllWriteUser) if err != nil { return err } @@ -432,7 +432,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (err error) { // write header file path = fmt.Sprintf("%s.part000", srcFile) - if err := os.WriteFile(path, jsonData, helpers.WriteUserReadAll); err != nil { + if err := os.WriteFile(path, jsonData, helpers.ReadAllWriteUser); err != nil { return fmt.Errorf("unable to write the file %s: %w", path, err) } fileNames = append(fileNames, path) diff --git a/src/pkg/utils/network.go b/src/pkg/utils/network.go index a4446d4d70..bf9c4c18a6 100644 --- a/src/pkg/utils/network.go +++ b/src/pkg/utils/network.go @@ -48,7 +48,7 @@ func DownloadToFile(src string, dst string, cosignKeyPath string) (err error) { return err } - err = CreateDirectory(filepath.Dir(dst), helpers.ReadWriteXUser) + err = CreateDirectory(filepath.Dir(dst), helpers.ReadWriteExecuteUser) if err != nil { return fmt.Errorf(lang.ErrCreatingDir, filepath.Dir(dst), err.Error()) } From d80c3a1968a622eed1a5a042c65cc825a3809e05 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 12 Feb 2024 21:30:30 +0000 Subject: [PATCH 086/113] using oras remote logger in zoci, composing transport with roundtripper --- src/pkg/oci/common.go | 7 ++++++- src/pkg/utils/helpers/transport.go | 8 ++++---- src/pkg/zoci/copier.go | 2 +- src/pkg/zoci/pull.go | 3 +-- src/pkg/zoci/push.go | 2 +- src/pkg/zoci/utils.go | 5 +---- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 495384cb42..14aea085e5 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -46,7 +46,7 @@ func WithPlainHTTP(plainHTTP bool) Modifier { // WithInsecureSkipVerify sets the insecure TLS flag for the remote func WithInsecureSkipVerify(insecure bool) Modifier { return func(o *OrasRemote) { - o.Transport.Base.(*http.Transport).TLSClientConfig.InsecureSkipVerify = insecure + o.Transport.RoundTripper.(*http.Transport).TLSClientConfig.InsecureSkipVerify = insecure } } @@ -106,6 +106,11 @@ func (o *OrasRemote) Repo() *remote.Repository { return o.repo } +// Repo gives you access to the underlying remote repository +func (o *OrasRemote) Log() *slog.Logger { + return &o.log +} + // setRepository sets the repository for the remote as well as the auth client. func (o *OrasRemote) setRepository(ref registry.Reference) error { o.root = nil diff --git a/src/pkg/utils/helpers/transport.go b/src/pkg/utils/helpers/transport.go index 577fe77ed0..f984019a76 100644 --- a/src/pkg/utils/helpers/transport.go +++ b/src/pkg/utils/helpers/transport.go @@ -15,15 +15,15 @@ import ( // Transport is an http.RoundTripper that keeps track of the in-flight // request and add hooks to report upload progress. type Transport struct { - Base http.RoundTripper + http.RoundTripper ProgressBar ProgressWriter } // NewTransport returns a custom transport that tracks an http.RoundTripper and a message.ProgressBar. func NewTransport(base http.RoundTripper, bar ProgressWriter) *Transport { return &Transport{ - Base: base, - ProgressBar: bar, + RoundTripper: base, + ProgressBar: bar, } } @@ -85,7 +85,7 @@ func (t *Transport) roundTrip(req *http.Request) (resp *http.Response, err error req.Body = io.NopCloser(io.TeeReader(req.Body, t.ProgressBar)) } - resp, err = t.Base.RoundTrip(req) + resp, err = t.RoundTrip(req) if resp != nil && req.Method == http.MethodHead && err == nil && t.ProgressBar != nil { if resp.ContentLength > 0 { diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index 0586dc3395..2e6f209f90 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -54,6 +54,6 @@ func CopyPackage(ctx context.Context, src *Remote, dst *Remote, concurrency int) return err } - message.Infof("Published %s to %s", src.Repo().Reference, dst.Repo().Reference) + src.Log().Info(fmt.Sprintf("Published %s to %s", src.Repo().Reference, dst.Repo().Reference)) return nil } diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index 6e357fc720..defcdff8b3 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -11,7 +11,6 @@ import ( "slices" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -42,7 +41,7 @@ var ( // - zarf.yaml.sig func (r *Remote) PullPackage(ctx context.Context, destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) ([]ocispec.Descriptor, error) { isPartialPull := len(layersToPull) > 0 - message.Debugf("Pulling %s", r.Repo().Reference) + r.Log().Debug(fmt.Sprintf("Pulling %s", r.Repo().Reference)) manifest, err := r.FetchRoot(ctx) if err != nil { diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 617922b92e..416ef6ab77 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -26,7 +26,7 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat } defer src.Close() - message.Infof("Publishing package to %s", r.Repo().Reference) + r.Log().Info(fmt.Sprintf("Publishing package to %s", r.Repo().Reference)) spinner := message.NewProgressSpinner("") defer spinner.Stop() diff --git a/src/pkg/zoci/utils.go b/src/pkg/zoci/utils.go index 2683ef6ab5..c0a8fc944a 100644 --- a/src/pkg/zoci/utils.go +++ b/src/pkg/zoci/utils.go @@ -9,7 +9,6 @@ import ( "fmt" "strings" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "oras.land/oras-go/v2/registry" @@ -34,11 +33,9 @@ func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata raw = fmt.Sprintf("%s-%s", raw, build.Flavor) } - message.Debug("Raw OCI reference from metadata:", raw) - ref, err := registry.ParseReference(raw) if err != nil { - return "", err + return "", fmt.Errorf("failed to parse %s: %w", raw, err) } return ref.String(), nil From 876f73e29b912ee1fd5f98db4d0123f9dd1fcbca Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 13 Feb 2024 13:45:01 +0000 Subject: [PATCH 087/113] re-adding base --- src/pkg/oci/common.go | 2 +- src/pkg/utils/helpers/transport.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 14aea085e5..56f11e3522 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -46,7 +46,7 @@ func WithPlainHTTP(plainHTTP bool) Modifier { // WithInsecureSkipVerify sets the insecure TLS flag for the remote func WithInsecureSkipVerify(insecure bool) Modifier { return func(o *OrasRemote) { - o.Transport.RoundTripper.(*http.Transport).TLSClientConfig.InsecureSkipVerify = insecure + o.Transport.Base.(*http.Transport).TLSClientConfig.InsecureSkipVerify = insecure } } diff --git a/src/pkg/utils/helpers/transport.go b/src/pkg/utils/helpers/transport.go index f984019a76..360c0e5f0c 100644 --- a/src/pkg/utils/helpers/transport.go +++ b/src/pkg/utils/helpers/transport.go @@ -15,15 +15,15 @@ import ( // Transport is an http.RoundTripper that keeps track of the in-flight // request and add hooks to report upload progress. type Transport struct { - http.RoundTripper + Base http.RoundTripper ProgressBar ProgressWriter } // NewTransport returns a custom transport that tracks an http.RoundTripper and a message.ProgressBar. func NewTransport(base http.RoundTripper, bar ProgressWriter) *Transport { return &Transport{ - RoundTripper: base, - ProgressBar: bar, + Base: base, + ProgressBar: bar, } } From 7aad1173dc7378eb0b2a2291530c33e2246f4753 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 13 Feb 2024 14:13:18 +0000 Subject: [PATCH 088/113] lint --- src/pkg/oci/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 56f11e3522..95821e82b7 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -106,7 +106,7 @@ func (o *OrasRemote) Repo() *remote.Repository { return o.repo } -// Repo gives you access to the underlying remote repository +// Log gives you access to the OrasRemote logger func (o *OrasRemote) Log() *slog.Logger { return &o.log } From 543f001efab5ae28f0a50a41b39b77a8e8198547 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 13 Feb 2024 14:14:38 +0000 Subject: [PATCH 089/113] make logger a pointer --- src/pkg/oci/common.go | 8 ++++---- src/pkg/zoci/common.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 95821e82b7..93c50e0b3d 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -30,7 +30,7 @@ type OrasRemote struct { root *Manifest Transport *helpers.Transport targetPlatform *ocispec.Platform - log slog.Logger + log *slog.Logger } // Modifier is a function that modifies an OrasRemote @@ -66,7 +66,7 @@ func WithUserAgent(userAgent string) Modifier { } // WithLogger sets the logger for the remote -func WithLogger(logger slog.Logger) Modifier { +func WithLogger(logger *slog.Logger) Modifier { return func(o *OrasRemote) { o.log = logger } @@ -87,7 +87,7 @@ func NewOrasRemote(url string, platform ocispec.Platform, mods ...Modifier) (*Or repo: &remote.Repository{Client: client}, Transport: helpers.NewTransport(transport, nil), targetPlatform: &platform, - log: *slog.Default(), + log: slog.Default(), } for _, mod := range mods { @@ -108,7 +108,7 @@ func (o *OrasRemote) Repo() *remote.Repository { // Log gives you access to the OrasRemote logger func (o *OrasRemote) Log() *slog.Logger { - return &o.log + return o.log } // setRepository sets the repository for the remote as well as the auth client. diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index da7bb9170f..e210f8c9c1 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -30,7 +30,7 @@ type Remote struct { // NewRemote returns an oras remote repository client and context for the given url // with zarf opination embedded func NewRemote(url string, platform ocispec.Platform, mods ...oci.Modifier) (*Remote, error) { - logger := *slog.New(message.ZarfHandler{}) + logger := slog.New(message.ZarfHandler{}) modifiers := append([]oci.Modifier{ oci.WithPlainHTTP(config.CommonOptions.Insecure), oci.WithInsecureSkipVerify(config.CommonOptions.Insecure), From c81ca36751c3db2b1dc15b059603c0a5bbde8352 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 13 Feb 2024 14:44:51 +0000 Subject: [PATCH 090/113] turning log.go into progress.go --- src/pkg/utils/helpers/{log.go => progres.go} | 25 -------------------- 1 file changed, 25 deletions(-) rename src/pkg/utils/helpers/{log.go => progres.go} (55%) diff --git a/src/pkg/utils/helpers/log.go b/src/pkg/utils/helpers/progres.go similarity index 55% rename from src/pkg/utils/helpers/log.go rename to src/pkg/utils/helpers/progres.go index 7b4be4fef7..35b416b755 100644 --- a/src/pkg/utils/helpers/log.go +++ b/src/pkg/utils/helpers/progres.go @@ -6,22 +6,6 @@ package helpers import "io" -// Debug does nothing -func (l *DiscardLogger) Debug(_ string, _ ...any) {} - -// Info does nothing -func (l *DiscardLogger) Info(_ string, _ ...any) {} - -// Warn does nothing -func (l *DiscardLogger) Warn(_ string, _ ...any) {} - -// Error does nothing -func (l *DiscardLogger) Error(_ string, _ ...any) {} - -// DiscardLogger is the default if the WithLogger modifier is not used -// It discards all logs -type DiscardLogger struct{} - // Write doesn't do anything but satisfy implementation func (DiscardProgressWriter) Write(p []byte) (int, error) { return len(p), nil @@ -34,15 +18,6 @@ func (DiscardProgressWriter) UpdateTitle(_ string) {} // Use this or nil or if you don't care about writing progress type DiscardProgressWriter struct{} -// Logger allows for varying levels of logging depending on importance -// Arguments are intended to be used in printf-style -type Logger interface { - Debug(msg string, args ...any) - Info(msg string, args ...any) - Warn(msg string, args ...any) - Error(msg string, args ...any) -} - // ProgressWriter wraps io.Writer, but also includes an updateTitle function to give the user // additional context on what's going on. Useful in OCI for tracking layers type ProgressWriter interface { From dd68dba09af30bada1a95da6797a696a224f00b1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 13 Feb 2024 14:46:49 +0000 Subject: [PATCH 091/113] fix roundtripper --- src/pkg/utils/helpers/transport.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/utils/helpers/transport.go b/src/pkg/utils/helpers/transport.go index 360c0e5f0c..577fe77ed0 100644 --- a/src/pkg/utils/helpers/transport.go +++ b/src/pkg/utils/helpers/transport.go @@ -85,7 +85,7 @@ func (t *Transport) roundTrip(req *http.Request) (resp *http.Response, err error req.Body = io.NopCloser(io.TeeReader(req.Body, t.ProgressBar)) } - resp, err = t.RoundTrip(req) + resp, err = t.Base.RoundTrip(req) if resp != nil && req.Method == http.MethodHead && err == nil && t.ProgressBar != nil { if resp.ContentLength > 0 { From 338f0ac10a66058efb6504f196ee45a13890f3ce Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 13 Feb 2024 14:47:26 +0000 Subject: [PATCH 092/113] rename --- src/pkg/utils/helpers/transport.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/utils/helpers/transport.go b/src/pkg/utils/helpers/transport.go index 577fe77ed0..5fa452ec10 100644 --- a/src/pkg/utils/helpers/transport.go +++ b/src/pkg/utils/helpers/transport.go @@ -90,8 +90,8 @@ func (t *Transport) roundTrip(req *http.Request) (resp *http.Response, err error if resp != nil && req.Method == http.MethodHead && err == nil && t.ProgressBar != nil { if resp.ContentLength > 0 { contentLength := int(resp.ContentLength) - data := make([]byte, contentLength) - t.ProgressBar.Write(data) + b := make([]byte, contentLength) + t.ProgressBar.Write(b) } } return resp, err From 2956b475fc59f255c49748145ecf5bbb0dd36830 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 13 Feb 2024 14:59:15 +0000 Subject: [PATCH 093/113] simplified layersfromrequestedcomponents --- src/pkg/zoci/pull.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index defcdff8b3..d6dde677e6 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -94,7 +94,6 @@ func (r *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom if err != nil { return nil, err } - images := map[string]bool{} tarballFormat := "%s.tar" for _, name := range requestedComponents { component := helpers.Find(pkg.Components, func(component types.ZarfComponent) bool { @@ -104,12 +103,11 @@ func (r *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom return nil, fmt.Errorf("component %s does not exist in this package", name) } } + var images []string for _, component := range pkg.Components { // If we requested this component, or it is required, we need to pull its images and tarball if slices.Contains(requestedComponents, component.Name) || component.Required { - for _, image := range component.Images { - images[image] = true - } + images = append(images, component.Images...) layers = append(layers, root.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf(tarballFormat, component.Name)))) } } @@ -127,7 +125,7 @@ func (r *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom if err != nil { return nil, err } - for image := range images { + for _, image := range images { // use docker's transform lib to parse the image ref // this properly mirrors the logic within create refInfo, err := transform.ParseImageRef(image) From ef5b27096e0c357cfcedca95bf51aa9989222007 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 13 Feb 2024 15:27:54 +0000 Subject: [PATCH 094/113] unneeded code in test --- src/test/e2e/50_oci_publish_deploy_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 8606550c7a..220c3123b7 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -129,10 +129,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - src, err := zoci.NewRemote(ref, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true), oci.WithInsecureSkipVerify(true)) + src, err := zoci.NewRemote(ref, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true)) suite.NoError(err) - dst, err := zoci.NewRemote(dstRef, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true), oci.WithInsecureSkipVerify(true)) + dst, err := zoci.NewRemote(dstRef, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) From 7ed05a6ecd111989195d1ea3423255f2f241d723 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 14 Feb 2024 18:52:24 +0000 Subject: [PATCH 095/113] added media type to index --- src/pkg/oci/push.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 01ff9b711a..9e4155272d 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -89,6 +89,7 @@ func (o *OrasRemote) UpdateIndex(ctx context.Context, tag string, publishedDesc if err != nil { if errors.Is(err, errdef.ErrNotFound) { index = ocispec.Index{ + MediaType: ocispec.MediaTypeImageIndex, Versioned: specs.Versioned{ SchemaVersion: 2, }, From 86486b9c7c1c8df44f5acd0acbfdd1cf1167fb97 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 16 Feb 2024 19:29:41 +0000 Subject: [PATCH 096/113] lint --- src/cmd/initialize.go | 2 +- src/cmd/tools/zarf.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 79452967db..4545f9018b 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -34,7 +34,7 @@ var initCmd = &cobra.Command{ Short: lang.CmdInitShort, Long: lang.CmdInitLong, Example: lang.CmdInitExample, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { zarfLogo := message.GetLogo() _, _ = fmt.Fprintln(os.Stderr, zarfLogo) diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index f4b7574f3e..e86d66f1ec 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -50,7 +50,7 @@ var getCredsCmd = &cobra.Command{ Example: lang.CmdToolsGetCredsExample, Aliases: []string{"gc"}, Args: cobra.MaximumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { state, err := cluster.NewClusterOrDie().LoadZarfState() if err != nil || state.Distro == "" { // If no distro the zarf secret did not load properly @@ -170,7 +170,7 @@ var clearCacheCmd = &cobra.Command{ Use: "clear-cache", Aliases: []string{"c"}, Short: lang.CmdToolsClearCacheShort, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { message.Notef(lang.CmdToolsClearCacheDir, config.GetAbsCachePath()) if err := os.RemoveAll(config.GetAbsCachePath()); err != nil { message.Fatalf(err, lang.CmdToolsClearCacheErr, config.GetAbsCachePath()) @@ -182,7 +182,7 @@ var clearCacheCmd = &cobra.Command{ var downloadInitCmd = &cobra.Command{ Use: "download-init", Short: lang.CmdToolsDownloadInitShort, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { url := zoci.GetInitPackageURL(config.CLIVersion) remote, err := zoci.NewRemote(url, oci.PlatformForArch(config.GetArch())) @@ -204,7 +204,7 @@ var generatePKICmd = &cobra.Command{ Aliases: []string{"pki"}, Short: lang.CmdToolsGenPkiShort, Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { pki := pki.GeneratePKI(args[0], subAltNames...) if err := os.WriteFile("tls.ca", pki.CA, helpers.ReadAllWriteUser); err != nil { message.Fatalf(err, lang.ErrWritingFile, "tls.ca", err.Error()) @@ -223,7 +223,7 @@ var generateKeyCmd = &cobra.Command{ Use: "gen-key", Aliases: []string{"key"}, Short: lang.CmdToolsGenKeyShort, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { // Utility function to prompt the user for the password to the private key passwordFunc := func(bool) ([]byte, error) { // perform the first prompt From b604441070c5d33a78e83c64bcce61de252a4865 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 20 Feb 2024 17:09:13 +0000 Subject: [PATCH 097/113] update title --- src/pkg/oci/common.go | 2 +- src/pkg/oci/copier.go | 2 +- src/pkg/oci/fetch.go | 2 +- src/pkg/oci/manifest.go | 2 +- src/pkg/oci/progress.go | 2 +- src/pkg/oci/pull.go | 2 +- src/pkg/oci/push.go | 2 +- src/pkg/oci/utils.go | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 93c50e0b3d..054d8b93d0 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package oci contains functions for interacting with artifacts stored in OCI registries. package oci import ( diff --git a/src/pkg/oci/copier.go b/src/pkg/oci/copier.go index e74eb9fd0a..99545449ae 100644 --- a/src/pkg/oci/copier.go +++ b/src/pkg/oci/copier.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package oci contains functions for interacting with artifacts stored in OCI registries. package oci import ( diff --git a/src/pkg/oci/fetch.go b/src/pkg/oci/fetch.go index 952fde58d8..332e256cef 100644 --- a/src/pkg/oci/fetch.go +++ b/src/pkg/oci/fetch.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package oci contains functions for interacting with artifacts stored in OCI registries. package oci import ( diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index 80f2999636..750edb64ef 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package oci contains functions for interacting with artifacts stored in OCI registries. package oci import ( diff --git a/src/pkg/oci/progress.go b/src/pkg/oci/progress.go index e2754bc60c..d245831ec4 100644 --- a/src/pkg/oci/progress.go +++ b/src/pkg/oci/progress.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package oci contains functions for interacting with artifacts stored in OCI registries. package oci import ( diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 2a1fe8f448..e47f4711f0 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package oci contains functions for interacting with artifacts stored in OCI registries. package oci import ( diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 9e4155272d..3b421b439d 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package oci contains functions for interacting with artifacts stored in OCI registries. package oci import ( diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index 417ee26796..568faf39a5 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +// Package oci contains functions for interacting with artifacts stored in OCI registries. package oci import ( From f97c833c8bdef645146c906a91f74dfd45768d86 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 21 Feb 2024 18:40:54 +0000 Subject: [PATCH 098/113] rename --- src/pkg/oci/push.go | 5 +++-- src/pkg/zoci/push.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 3b421b439d..8f43fa15b5 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -57,8 +57,9 @@ func (o *OrasRemote) PushManifestConfigFromMetadata(ctx context.Context, annotat return o.PushLayer(ctx, manifestConfigBytes, configMediaType) } -// GeneratePackManifest generates an OCI Image Manifest based on the given parameters -func (o *OrasRemote) GeneratePackManifest(ctx context.Context, src *file.Store, descs []ocispec.Descriptor, +// CreateAndPushManifest generates an OCI Image Manifest based on the given parameters +// pushes that manifest to the remote repository and returns the manifest descriptor. +func (o *OrasRemote) CreateAndPushManifest(ctx context.Context, src *file.Store, descs []ocispec.Descriptor, configDesc *ocispec.Descriptor, annotations map[string]string) (ocispec.Descriptor, error) { packOpts := oras.PackManifestOptions{ Layers: descs, diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 416ef6ab77..2571ac88d8 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -62,7 +62,7 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat if err != nil { return err } - root, err := r.GeneratePackManifest(ctx, src, descs, manifestConfigDesc, annotations) + root, err := r.CreateAndPushManifest(ctx, src, descs, manifestConfigDesc, annotations) if err != nil { return err } From 0a980cfbf48e40824b19f1132ec8a157feb16511 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 21 Feb 2024 18:44:58 +0000 Subject: [PATCH 099/113] lint --- src/cmd/tools/zarf.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index e86d66f1ec..22f76bdee0 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -37,7 +37,7 @@ var deprecatedGetGitCredsCmd = &cobra.Command{ Hidden: true, Short: lang.CmdToolsGetGitPasswdShort, Long: lang.CmdToolsGetGitPasswdLong, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { message.Warn(lang.CmdToolsGetGitPasswdDeprecation) getCredsCmd.Run(getCredsCmd, []string{"git"}) }, @@ -170,7 +170,7 @@ var clearCacheCmd = &cobra.Command{ Use: "clear-cache", Aliases: []string{"c"}, Short: lang.CmdToolsClearCacheShort, - Run: func(_ *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { message.Notef(lang.CmdToolsClearCacheDir, config.GetAbsCachePath()) if err := os.RemoveAll(config.GetAbsCachePath()); err != nil { message.Fatalf(err, lang.CmdToolsClearCacheErr, config.GetAbsCachePath()) From 443fda05f85fddd78337264cdb2b6946496c4dae Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:25:57 -0500 Subject: [PATCH 100/113] Apply suggestions from code review Co-authored-by: Lucas Rodriguez --- src/pkg/oci/common.go | 2 +- src/pkg/zoci/fetch.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 054d8b93d0..3cb94c0a0b 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -58,7 +58,7 @@ func PlatformForArch(arch string) ocispec.Platform { } } -// WithUserAgent sets the target architecture for the remote +// WithUserAgent sets the user agent for the remote func WithUserAgent(userAgent string) Modifier { return func(o *OrasRemote) { o.repo.Client.(*auth.Client).SetUserAgent(userAgent) diff --git a/src/pkg/zoci/fetch.go b/src/pkg/zoci/fetch.go index f9684be74e..2d6068bd57 100644 --- a/src/pkg/zoci/fetch.go +++ b/src/pkg/zoci/fetch.go @@ -26,7 +26,7 @@ func (r *Remote) FetchZarfYAML(ctx context.Context) (pkg types.ZarfPackage, err func (r *Remote) FetchImagesIndex(ctx context.Context) (index *ocispec.Index, err error) { manifest, err := r.FetchRoot(ctx) if err != nil { - return index, err + return nil, err } return oci.FetchJSONFile[*ocispec.Index](ctx, r.FetchLayer, manifest, ZarfPackageIndexPath) } From cca1703b922ffeedf6cbe444f2462d359ca7c4c8 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 22 Feb 2024 13:29:27 +0000 Subject: [PATCH 101/113] moving variables to layout --- src/pkg/layout/constants.go | 11 +++++++++++ src/pkg/layout/package.go | 6 +++--- src/pkg/zoci/fetch.go | 2 +- src/pkg/zoci/pull.go | 14 ++++---------- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/pkg/layout/constants.go b/src/pkg/layout/constants.go index 9c178ac961..bb260f6a76 100644 --- a/src/pkg/layout/constants.go +++ b/src/pkg/layout/constants.go @@ -4,6 +4,8 @@ // Package layout contains functions for interacting with Zarf's package layout on disk. package layout +import "path/filepath" + // Constants used in the default package layout. const ( TempDir = "temp" @@ -31,3 +33,12 @@ const ( InjectorBinary = "zarf-injector" InjectorPayloadTarGz = "payload.tgz" ) + +var ( + // IndexPath is the path to the index.json file + IndexPath = filepath.Join(ImagesDir, IndexJSON) + // ImagesBlobsDir is the path to the directory containing the image blobs in the OCI package. + ImagesBlobsDir = filepath.Join(ImagesDir, "blobs", "sha256") + // OCILayoutPath is the path to the oci-layout file + OCILayoutPath = filepath.Join(ImagesDir, OCILayout) +) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index a9dfb36b97..4e7bcff63a 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -187,11 +187,11 @@ func (pp *PackagePaths) SetFromPaths(paths []string) { pp.Checksums = filepath.Join(pp.Base, path) case path == SBOMTar: pp.SBOMs.Path = filepath.Join(pp.Base, path) - case path == filepath.Join(ImagesDir, OCILayout): + case path == OCILayoutPath: pp.Images.OCILayout = filepath.Join(pp.Base, path) - case path == filepath.Join(ImagesDir, IndexJSON): + case path == IndexPath: pp.Images.Index = filepath.Join(pp.Base, path) - case strings.HasPrefix(path, filepath.Join(ImagesDir, "blobs", "sha256")): + case strings.HasPrefix(path, ImagesBlobsDir): if pp.Images.Base == "" { pp.Images.Base = filepath.Join(pp.Base, ImagesDir) } diff --git a/src/pkg/zoci/fetch.go b/src/pkg/zoci/fetch.go index f9684be74e..95aa50e3f3 100644 --- a/src/pkg/zoci/fetch.go +++ b/src/pkg/zoci/fetch.go @@ -28,5 +28,5 @@ func (r *Remote) FetchImagesIndex(ctx context.Context) (index *ocispec.Index, er if err != nil { return index, err } - return oci.FetchJSONFile[*ocispec.Index](ctx, r.FetchLayer, manifest, ZarfPackageIndexPath) + return oci.FetchJSONFile[*ocispec.Index](ctx, r.FetchLayer, manifest, layout.IndexPath) } diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index d6dde677e6..f44fafb2f9 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -23,12 +23,6 @@ import ( var ( // PackageAlwaysPull is a list of paths that will always be pulled from the remote repository. PackageAlwaysPull = []string{layout.ZarfYAML, layout.Checksums, layout.Signature} - // ZarfPackageIndexPath is the path to the index.json file in the OCI package. - ZarfPackageIndexPath = filepath.Join("images", "index.json") - // ZarfPackageImagesBlobsDir is the path to the directory containing the image blobs in the OCI package. - ZarfPackageImagesBlobsDir = filepath.Join("images", "blobs", "sha256") - // ZarfPackageLayoutPath is the path to the oci-layout file in the OCI package. - ZarfPackageLayoutPath = filepath.Join("images", "oci-layout") ) // PullPackage pulls the package from the remote repository and saves it to the given path. @@ -120,7 +114,7 @@ func (r *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom } if len(images) > 0 { // Add the image index and the oci-layout layers - layers = append(layers, root.Locate(ZarfPackageIndexPath), root.Locate(ZarfPackageLayoutPath)) + layers = append(layers, root.Locate(layout.IndexPath), root.Locate(layout.OCILayoutPath)) index, err := r.FetchImagesIndex(ctx) if err != nil { return nil, err @@ -147,12 +141,12 @@ func (r *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom return nil, err } // Add the manifest and the manifest config layers - layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifestDescriptor.Digest.Encoded()))) - layers = append(layers, root.Locate(filepath.Join(ZarfPackageImagesBlobsDir, manifest.Config.Digest.Encoded()))) + layers = append(layers, root.Locate(filepath.Join(layout.ImagesBlobsDir, manifestDescriptor.Digest.Encoded()))) + layers = append(layers, root.Locate(filepath.Join(layout.ImagesBlobsDir, manifest.Config.Digest.Encoded()))) // Add all the layers from the manifest for _, layer := range manifest.Layers { - layerPath := filepath.Join(ZarfPackageImagesBlobsDir, layer.Digest.Encoded()) + layerPath := filepath.Join(layout.ImagesBlobsDir, layer.Digest.Encoded()) layers = append(layers, root.Locate(layerPath)) } } From 679eaf390074e7a6e2faa30166eb83f8a0ae1567 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 22 Feb 2024 13:36:35 +0000 Subject: [PATCH 102/113] fix --- src/cmd/tools/zarf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index d21ea7b371..5f8da77706 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -183,7 +183,7 @@ var downloadInitCmd = &cobra.Command{ Use: "download-init", Short: lang.CmdToolsDownloadInitShort, Run: func(_ *cobra.Command, _ []string) { - url := oci.GetInitPackageURL(config.CLIVersion) + url := zoci.GetInitPackageURL(config.CLIVersion) remote, err := zoci.NewRemote(url, oci.PlatformForArch(config.GetArch())) if err != nil { From ca2a018d0c699f6c28d4d0ba899229d50ea662ba Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 26 Feb 2024 17:37:53 +0000 Subject: [PATCH 103/113] lint --- src/pkg/message/handler.go | 2 +- src/pkg/packager/publish.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pkg/message/handler.go b/src/pkg/message/handler.go index a2269cbf2e..46fab9cbcf 100644 --- a/src/pkg/message/handler.go +++ b/src/pkg/message/handler.go @@ -29,7 +29,7 @@ func (z ZarfHandler) WithGroup(_ string) slog.Handler { // Handle prints the respective logging function in zarf // This function ignores any key pairs passed through the record -func (z ZarfHandler) Handle(ctx context.Context, record slog.Record) error { +func (z ZarfHandler) Handle(_ context.Context, record slog.Record) error { level := record.Level message := record.Message diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 21c8d9c270..ae368a7afa 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -42,10 +42,7 @@ func (p *Packager) Publish() (err error) { return err } - if err := zoci.CopyPackage(ctx, srcRemote, dstRemote, config.CommonOptions.OCIConcurrency); err != nil { - return err - } - return nil + return zoci.CopyPackage(ctx, srcRemote, dstRemote, config.CommonOptions.OCIConcurrency) } if p.cfg.CreateOpts.IsSkeleton { From 9943d612de784a89e1aa56d45495de333e6944c1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 26 Feb 2024 18:20:08 +0000 Subject: [PATCH 104/113] spelling --- src/pkg/utils/helpers/io.go | 3 +-- src/pkg/utils/helpers/{progres.go => progress.go} | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename src/pkg/utils/helpers/{progres.go => progress.go} (100%) diff --git a/src/pkg/utils/helpers/io.go b/src/pkg/utils/helpers/io.go index 726ff94fa5..e0b1879f86 100644 --- a/src/pkg/utils/helpers/io.go +++ b/src/pkg/utils/helpers/io.go @@ -5,14 +5,13 @@ package helpers const ( - // ReadWriteUser is used for any internal file not normallly used by the end user or containing senstive data + // ReadWriteUser is used for any internal file not normally used by the end user or containing sensitive data ReadWriteUser = 0600 // ReadAllWriteUser is used for any non sensitive file intended to be consumed by the end user ReadAllWriteUser = 0644 // ReadWriteExecuteUser is used for any directory or executable not normally used by the end user or containing sensitive data ReadWriteExecuteUser = 0700 - // ReadExecuteAllWriteUser is used for any non sensitive directory or executable intended to be consumed by the end user ReadExecuteAllWriteUser = 0755 ) diff --git a/src/pkg/utils/helpers/progres.go b/src/pkg/utils/helpers/progress.go similarity index 100% rename from src/pkg/utils/helpers/progres.go rename to src/pkg/utils/helpers/progress.go From fe9577d9bfcf457d4fcdf7e5225ae598ee8b30a8 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 27 Feb 2024 17:43:03 +0000 Subject: [PATCH 105/113] rename --- src/pkg/oci/push.go | 4 ++-- src/pkg/zoci/push.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 8f43fa15b5..3793369c3c 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -39,8 +39,8 @@ func (o *OrasRemote) PushLayer(ctx context.Context, b []byte, mediaType string) return &desc, o.repo.Push(ctx, desc, bytes.NewReader(b)) } -// PushManifestConfigFromMetadata pushes the manifest config with metadata to the remote repository. -func (o *OrasRemote) PushManifestConfigFromMetadata(ctx context.Context, annotations map[string]string, configMediaType string) (*ocispec.Descriptor, error) { +// CreateAndPushManifestConfig pushes the manifest config with metadata to the remote repository. +func (o *OrasRemote) CreateAndPushManifestConfig(ctx context.Context, annotations map[string]string, configMediaType string) (*ocispec.Descriptor, error) { if annotations[ocispec.AnnotationTitle] == "" { return nil, fmt.Errorf("invalid annotations: please include value for %q", ocispec.AnnotationTitle) } diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 2571ac88d8..c45e14045d 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -58,7 +58,7 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat // push the manifest config // since this config is so tiny, and the content is not used again // it is not logged to the progress, but will error if it fails - manifestConfigDesc, err := r.PushManifestConfigFromMetadata(ctx, annotations, ZarfConfigMediaType) + manifestConfigDesc, err := r.CreateAndPushManifestConfig(ctx, annotations, ZarfConfigMediaType) if err != nil { return err } From f090179b62499a76374fe35eb99da598e09858ce Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 27 Feb 2024 19:21:04 +0000 Subject: [PATCH 106/113] function rename --- src/pkg/oci/push.go | 4 ++-- src/pkg/zoci/push.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 3793369c3c..7e028c2023 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -57,9 +57,9 @@ func (o *OrasRemote) CreateAndPushManifestConfig(ctx context.Context, annotation return o.PushLayer(ctx, manifestConfigBytes, configMediaType) } -// CreateAndPushManifest generates an OCI Image Manifest based on the given parameters +// PackAndTagManifest generates an OCI Image Manifest based on the given parameters // pushes that manifest to the remote repository and returns the manifest descriptor. -func (o *OrasRemote) CreateAndPushManifest(ctx context.Context, src *file.Store, descs []ocispec.Descriptor, +func (o *OrasRemote) PackAndTagManifest(ctx context.Context, src *file.Store, descs []ocispec.Descriptor, configDesc *ocispec.Descriptor, annotations map[string]string) (ocispec.Descriptor, error) { packOpts := oras.PackManifestOptions{ Layers: descs, diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index c45e14045d..9bbf740f2d 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -62,11 +62,13 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat if err != nil { return err } - root, err := r.CreateAndPushManifest(ctx, src, descs, manifestConfigDesc, annotations) + root, err := r.PackAndTagManifest(ctx, src, descs, manifestConfigDesc, annotations) if err != nil { return err } + total += root.Size + manifestConfigDesc.Size + progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", r.Repo().Reference.Repository, r.Repo().Reference.Reference)) r.Transport.ProgressBar = progressBar From fc31c89d2d5de73d92dc7bcb5d3afe7fb9f6a83e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 27 Feb 2024 19:55:10 +0000 Subject: [PATCH 107/113] more correct progress total --- src/pkg/zoci/push.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 9bbf740f2d..57ab71e870 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -67,7 +67,7 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat return err } - total += root.Size + manifestConfigDesc.Size + total += root.Size progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", r.Repo().Reference.Repository, r.Repo().Reference.Reference)) r.Transport.ProgressBar = progressBar @@ -91,7 +91,6 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat } func annotationsFromMetadata(metadata *types.ZarfMetadata) map[string]string { - annotations := map[string]string{ ocispec.AnnotationTitle: metadata.Name, ocispec.AnnotationDescription: metadata.Description, From 765f152dbba5ce714078c669f9878925944cfef0 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 28 Feb 2024 16:42:30 +0000 Subject: [PATCH 108/113] update comments --- src/pkg/oci/manifest.go | 3 --- src/pkg/oci/push.go | 2 -- 2 files changed, 5 deletions(-) diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index 750edb64ef..6da4f6afff 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -13,9 +13,6 @@ import ( ) // Manifest is a wrapper around the OCI manifest -// -// it includes the path to the index.json, oci-layout, and image blobs. -// as well as a few helper functions for locating layers and calculating the size of the layers. type Manifest struct { ocispec.Manifest } diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 7e028c2023..cda4a8974f 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -25,8 +25,6 @@ import ( // which causes an error on Google Artifact Registry // // to negate this, we create a simple manifest config with some build metadata -// -// the contents of this file are not used by Zarf type ConfigPartial struct { Architecture string `json:"architecture"` OCIVersion string `json:"ociVersion"` From 332f32ff8b4099617f0787c2215d0cb873082d40 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 28 Feb 2024 19:51:01 +0000 Subject: [PATCH 109/113] moving out validate reference as it is unnecessary --- src/pkg/oci/utils.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index 568faf39a5..07a506d15a 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -5,13 +5,8 @@ package oci import ( - "fmt" - "strings" - - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" - "oras.land/oras-go/v2/registry" ) // IsEmptyDescriptor returns true if the given descriptor is empty. @@ -19,16 +14,6 @@ func IsEmptyDescriptor(desc ocispec.Descriptor) bool { return desc.Digest == "" && desc.Size == 0 } -// ValidateReference validates the given url is a valid OCI reference. -func ValidateReference(url string) error { - if !strings.HasPrefix(url, helpers.OCIURLPrefix) { - return fmt.Errorf("oci url reference must begin with %s", helpers.OCIURLPrefix) - } - sansPrefix := strings.TrimPrefix(url, helpers.OCIURLPrefix) - _, err := registry.ParseReference(sansPrefix) - return err -} - // RemoveDuplicateDescriptors removes duplicate descriptors from the given list. func RemoveDuplicateDescriptors(descriptors []ocispec.Descriptor) []ocispec.Descriptor { keys := make(map[string]bool) From 9d70650b7837efc40bc9964fdc3b69859cab48e1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 29 Feb 2024 21:33:40 +0000 Subject: [PATCH 110/113] correcting progres bar total --- src/pkg/zoci/push.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index 57ab71e870..c059de36c8 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -56,8 +56,6 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat r.Repo().SetReferrersCapability(false) // push the manifest config - // since this config is so tiny, and the content is not used again - // it is not logged to the progress, but will error if it fails manifestConfigDesc, err := r.CreateAndPushManifestConfig(ctx, annotations, ZarfConfigMediaType) if err != nil { return err @@ -67,7 +65,7 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat return err } - total += root.Size + total += manifestConfigDesc.Size progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", r.Repo().Reference.Repository, r.Repo().Reference.Reference)) r.Transport.ProgressBar = progressBar From ec545124f6501ef07c0901db882fde0f069d65bc Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 29 Feb 2024 21:46:41 +0000 Subject: [PATCH 111/113] fixing error progress bar logic in push --- src/pkg/zoci/push.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index c059de36c8..3bcd180561 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -68,6 +68,7 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat total += manifestConfigDesc.Size progressBar := message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", r.Repo().Reference.Repository, r.Repo().Reference.Reference)) + defer progressBar.Stop() r.Transport.ProgressBar = progressBar publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), r.Repo(), "", copyOpts) @@ -79,11 +80,6 @@ func (r *Remote) PublishPackage(ctx context.Context, pkg *types.ZarfPackage, pat return err } - if err != nil { - progressBar.Stop() - return fmt.Errorf("unable to publish package: %w", err) - } - progressBar.Successf("Published %s [%s]", r.Repo().Reference, ZarfLayerMediaTypeBlob) return nil } From f1f1439186fb1e2ab83a7efd49ce9a08198d9443 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 4 Mar 2024 13:37:15 +0000 Subject: [PATCH 112/113] switching back to map --- src/pkg/zoci/pull.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pkg/zoci/pull.go b/src/pkg/zoci/pull.go index f44fafb2f9..54aca7d218 100644 --- a/src/pkg/zoci/pull.go +++ b/src/pkg/zoci/pull.go @@ -97,11 +97,13 @@ func (r *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom return nil, fmt.Errorf("component %s does not exist in this package", name) } } - var images []string + images := map[string]bool{} for _, component := range pkg.Components { // If we requested this component, or it is required, we need to pull its images and tarball if slices.Contains(requestedComponents, component.Name) || component.Required { - images = append(images, component.Images...) + for _, image := range component.Images { + images[image] = true + } layers = append(layers, root.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf(tarballFormat, component.Name)))) } } @@ -119,7 +121,7 @@ func (r *Remote) LayersFromRequestedComponents(ctx context.Context, requestedCom if err != nil { return nil, err } - for _, image := range images { + for image := range images { // use docker's transform lib to parse the image ref // this properly mirrors the logic within create refInfo, err := transform.ParseImageRef(image) From ea06cddc8fad4930b855f878ff58554d28130dbd Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 4 Mar 2024 17:50:49 +0000 Subject: [PATCH 113/113] back to 1 media type! --- src/pkg/oci/push.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 28b14cf718..cda4a8974f 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -92,7 +92,6 @@ func (o *OrasRemote) UpdateIndex(ctx context.Context, tag string, publishedDesc Versioned: specs.Versioned{ SchemaVersion: 2, }, - MediaType: ocispec.MediaTypeImageIndex, Manifests: []ocispec.Descriptor{ { MediaType: ocispec.MediaTypeImageManifest,