diff --git a/src/pkg/bundle/deploy.go b/src/pkg/bundle/deploy.go index 3db04484..95bb9c14 100644 --- a/src/pkg/bundle/deploy.go +++ b/src/pkg/bundle/deploy.go @@ -125,7 +125,7 @@ func deployPackages(packages []types.Package, resume bool, b *Bundle) error { // Automatically confirm the package deployment zarfConfig.CommonOptions.Confirm = true - source, err := sources.New(b.cfg.DeployOpts.Source, pkg.Name, opts, sha, nsOverrides) + source, err := sources.New(b.cfg.DeployOpts.Source, pkg, opts, sha, nsOverrides) if err != nil { return err } diff --git a/src/pkg/bundle/publish.go b/src/pkg/bundle/publish.go index c2f0b1c0..659d04e6 100644 --- a/src/pkg/bundle/publish.go +++ b/src/pkg/bundle/publish.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/pkg/oci" "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" "github.com/defenseunicorns/zarf/src/pkg/zoci" av3 "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -19,7 +20,7 @@ import ( // Publish publishes a bundle to a remote OCI registry func (b *Bundle) Publish() error { - b.cfg.PublishOpts.Destination = utils.EnsureOCIPrefix(b.cfg.PublishOpts.Destination) + b.cfg.PublishOpts.Destination = boci.EnsureOCIPrefix(b.cfg.PublishOpts.Destination) // load bundle metadata into memory // todo: having the tmp dir be the provider.dst is weird diff --git a/src/pkg/bundle/remote.go b/src/pkg/bundle/remote.go index c87d8516..4d7017f7 100644 --- a/src/pkg/bundle/remote.go +++ b/src/pkg/bundle/remote.go @@ -7,18 +7,17 @@ package bundle import ( "bytes" "context" - "encoding/json" "errors" "fmt" "os" "path/filepath" "slices" - "strings" "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/pkg/oci" "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -173,37 +172,13 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U layersToPull = append(layersToPull, rootManifest.Config) for _, pkg := range bundle.Packages { - - // grab sha of zarf image manifest and pull it down - sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle - manifestDesc := rootManifest.Locate(sha) - manifestBytes, err := op.FetchLayer(ctx, manifestDesc) + // go through the pkg's layers and figure out which ones to pull based on the req'd + selected components + pkgLayers, estPkgBytes, err := boci.FindBundledPkgLayers(ctx, pkg, rootManifest, op.OrasRemote) if err != nil { return nil, nil, err } - - // unmarshal the zarf image manifest and add it to the layers to pull - var manifest oci.Manifest - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return nil, nil, err - } - layersToPull = append(layersToPull, manifestDesc) - progressBar := message.NewProgressBar(int64(len(manifest.Layers)), fmt.Sprintf("Verifying layers in Zarf package: %s", pkg.Name)) - - // go through the layers in the zarf image manifest and check if they exist in the remote - for _, layer := range manifest.Layers { - ok, err := op.Repo().Blobs().Exists(ctx, layer) - progressBar.Add(1) - estimatedBytes += layer.Size - if err != nil { - return nil, nil, err - } - // if the layer exists in the remote, add it to the layers to pull - if ok { - layersToPull = append(layersToPull, layer) - } - } - progressBar.Successf("Verified %s package", pkg.Name) + layersToPull = append(layersToPull, pkgLayers...) + estimatedBytes += estPkgBytes } store, err := ocistore.NewWithContext(ctx, op.dst) @@ -219,7 +194,7 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U layersToPull = append(layersToPull, rootDesc) // create copy options for oras.Copy() - copyOpts := utils.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) + copyOpts := boci.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) // Create a thread to update a progress bar as we save the package to disk doneSaving := make(chan error) @@ -255,7 +230,7 @@ func getOCIValidatedSource(source string) (string, error) { OS: oci.MultiOS, } // Check provided repository path - sourceWithOCI := utils.EnsureOCIPrefix(source) + sourceWithOCI := boci.EnsureOCIPrefix(source) remote, err := zoci.NewRemote(sourceWithOCI, platform) if err == nil { source = sourceWithOCI diff --git a/src/pkg/bundle/remove.go b/src/pkg/bundle/remove.go index 2e34ef2f..a6768064 100644 --- a/src/pkg/bundle/remove.go +++ b/src/pkg/bundle/remove.go @@ -94,7 +94,7 @@ func removePackages(packagesToRemove []types.Package, b *Bundle) error { } sha := strings.Split(pkg.Ref, "sha256:")[1] - source, err := sources.New(b.cfg.RemoveOpts.Source, pkg.Name, opts, sha, nil) + source, err := sources.New(b.cfg.RemoveOpts.Source, pkg, opts, sha, nil) if err != nil { return err } diff --git a/src/pkg/bundle/tarball.go b/src/pkg/bundle/tarball.go index 6553b4d9..6af34783 100644 --- a/src/pkg/bundle/tarball.go +++ b/src/pkg/bundle/tarball.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/pkg/oci" "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -217,6 +218,7 @@ func (tp *tarballBundleProvider) LoadBundleMetadata() (types.PathMap, error) { return loaded, nil } +// getZarfLayers returns the layers of the Zarf package that are in the bundle func (tp *tarballBundleProvider) getZarfLayers(store *ocistore.Store, pkgManifestDesc ocispec.Descriptor) ([]ocispec.Descriptor, int64, error) { var layersToPull []ocispec.Descriptor estimatedPkgSize := int64(0) @@ -278,7 +280,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o layersToPush = append(layersToPush, bundleRootManifest.Config) // copy bundle - copyOpts := utils.CreateCopyOpts(layersToPush, config.CommonOptions.OCIConcurrency) + copyOpts := boci.CreateCopyOpts(layersToPush, config.CommonOptions.OCIConcurrency) progressBar := message.NewProgressBar(estimatedBytes, fmt.Sprintf("Publishing %s:%s", remote.Repo().Reference.Repository, remote.Repo().Reference.Reference)) defer progressBar.Stop() remote.SetProgressWriter(progressBar) @@ -287,7 +289,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o ref := bundle.Metadata.Version // check for existing index - index, err := utils.GetIndex(remote, ref) + index, err := boci.GetIndex(remote, ref) if err != nil { return err } @@ -315,7 +317,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o } // create or update, then push index.json - err = utils.UpdateIndex(index, remote, &bundle, tp.bundleRootDesc) + err = boci.UpdateIndex(index, remote, &bundle, tp.bundleRootDesc) if err != nil { return err } diff --git a/src/pkg/bundler/common.go b/src/pkg/bundler/common.go index f1f6012d..b6a36dbd 100644 --- a/src/pkg/bundler/common.go +++ b/src/pkg/bundler/common.go @@ -11,7 +11,7 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/pkg/oci" - "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/zoci" @@ -55,7 +55,7 @@ func pushManifestConfigFromMetadata(r *oci.OrasRemote, metadata *types.UDSMetada OCIVersion: "1.0.1", Annotations: annotations, } - manifestConfigDesc, err := utils.ToOCIRemote(manifestConfig, zoci.ZarfLayerMediaTypeBlob, r) + manifestConfigDesc, err := boci.ToOCIRemote(manifestConfig, zoci.ZarfLayerMediaTypeBlob, r) if err != nil { return ocispec.Descriptor{}, err } diff --git a/src/pkg/bundler/fetcher/common.go b/src/pkg/bundler/fetcher/common.go new file mode 100644 index 00000000..023d3dc5 --- /dev/null +++ b/src/pkg/bundler/fetcher/common.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2023-Present The UDS Authors + +// Package fetcher contains functionality to fetch local and remote Zarf pkgs for local bundling +package fetcher + +import ( + "os" + "path/filepath" + "strings" + + "github.com/defenseunicorns/pkg/helpers" + "github.com/defenseunicorns/uds-cli/src/config" + "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" + zarfSources "github.com/defenseunicorns/zarf/src/pkg/packager/sources" + zarfTypes "github.com/defenseunicorns/zarf/src/types" + goyaml "github.com/goccy/go-yaml" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// loadPkg loads a package from a tarball source and filters out optional components +func loadPkg(pkgTmp string, pkgSrc zarfSources.PackageSource, optionalComponents []string) (zarfTypes.ZarfPackage, *layout.PackagePaths, error) { + // create empty layout and source + pkgPaths := layout.New(pkgTmp) + + // create filter for optional components + createFilter := filters.Combine( + filters.ForDeploy(strings.Join(optionalComponents, ","), false), + ) + + // load the package with the filter (calling LoadPackage populates the pkgPaths with the files from the tarball) + pkg, _, err := pkgSrc.LoadPackage(pkgPaths, createFilter, false) + if err != nil { + return zarfTypes.ZarfPackage{}, nil, err + } + return pkg, pkgPaths, nil +} + +// getImgLayerDigests grabs the digests of the layers from the images in the image index +func getImgLayerDigests(manifestsToInclude []ocispec.Descriptor, pkgPaths *layout.PackagePaths) ([]string, error) { + var includeLayers []string + for _, manifest := range manifestsToInclude { + includeLayers = append(includeLayers, manifest.Digest.Hex()) // be sure to include image manifest + manifestBytes, err := os.ReadFile(filepath.Join(pkgPaths.Images.Base, config.BlobsDir, manifest.Digest.Hex())) + if err != nil { + return nil, err + } + var imgManifest ocispec.Manifest + err = goyaml.Unmarshal(manifestBytes, &imgManifest) + if err != nil { + return nil, err + } + includeLayers = append(includeLayers, imgManifest.Config.Digest.Hex()) // don't forget the config + for _, layer := range imgManifest.Layers { + includeLayers = append(includeLayers, layer.Digest.Hex()) + } + } + return includeLayers, nil +} + +// filterPkgPaths grabs paths that either not in the blobs dir or are in includeLayers +func filterPkgPaths(pkgPaths *layout.PackagePaths, includeLayers []string, optionalComponents []zarfTypes.ZarfComponent) []string { + var filteredPaths []string + paths := pkgPaths.Files() + for _, path := range paths { + // include all paths that aren't in the blobs dir + if !strings.Contains(path, config.BlobsDir) { + // only grab req'd + specified optional components + if strings.Contains(path, "/components/") { + if shouldInclude := utils.IncludeComponent(path, optionalComponents); shouldInclude { + filteredPaths = append(filteredPaths, path) + continue + } + } else { + filteredPaths = append(filteredPaths, path) + } + } + // include paths that are in the blobs dir and are in includeLayers + for _, layer := range includeLayers { + if strings.Contains(path, config.BlobsDir) && strings.Contains(path, layer) { + filteredPaths = append(filteredPaths, path) + break + } + } + } + + // ensure zarf.yaml, checksums and SBOMS (if exists) are always included + // note we may have extra SBOMs because they are not filtered or modified + alwaysInclude := []string{pkgPaths.ZarfYAML, pkgPaths.Checksums} + if pkgPaths.SBOMs.Path != "" { + alwaysInclude = append(alwaysInclude, pkgPaths.SBOMs.Path) + } + filteredPaths = helpers.MergeSlices(filteredPaths, alwaysInclude, func(a, b string) bool { + return a == b + }) + + return filteredPaths +} diff --git a/src/pkg/bundler/fetcher/local.go b/src/pkg/bundler/fetcher/local.go index 88d14656..da538b07 100644 --- a/src/pkg/bundler/fetcher/local.go +++ b/src/pkg/bundler/fetcher/local.go @@ -6,7 +6,7 @@ package fetcher import ( "context" - "fmt" + "encoding/json" "io" "os" "path/filepath" @@ -14,13 +14,13 @@ import ( "github.com/defenseunicorns/pkg/oci" "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" + zarfSources "github.com/defenseunicorns/zarf/src/pkg/packager/sources" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/zoci" zarfTypes "github.com/defenseunicorns/zarf/src/types" - goyaml "github.com/goccy/go-yaml" - av3 "github.com/mholt/archiver/v3" av4 "github.com/mholt/archiver/v4" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -45,17 +45,7 @@ func (f *localFetcher) Fetch() ([]ocispec.Descriptor, error) { } f.extractDst = pkgTmp - err = f.extract() - if err != nil { - return nil, err - } - - zarfPkg, err := f.load() - if err != nil { - return nil, err - } - - layerDescs, err := f.toBundle(zarfPkg, pkgTmp) + layerDescs, err := f.toBundle(pkgTmp) if err != nil { return nil, err } @@ -65,6 +55,7 @@ func (f *localFetcher) Fetch() ([]ocispec.Descriptor, error) { // GetPkgMetadata grabs metadata from a local Zarf package's zarf.yaml func (f *localFetcher) GetPkgMetadata() (zarfTypes.ZarfPackage, error) { + // todo: can we refactor to use Zarf fns? tmpDir, err := zarfUtils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { return zarfTypes.ZarfPackage{}, err @@ -110,71 +101,76 @@ func (f *localFetcher) GetPkgMetadata() (zarfTypes.ZarfPackage, error) { return zarfYAML, err } -// extract extracts a compressed Zarf archive into a directory -func (f *localFetcher) extract() error { - err := av3.Unarchive(f.pkg.Path, f.extractDst) // todo: awkward to use old version of mholt/archiver - if err != nil { - return err - } - return nil -} - -// load loads a zarf.yaml into a Zarf object -func (f *localFetcher) load() (zarfTypes.ZarfPackage, error) { - // grab zarf.yaml from extracted archive - p, err := os.ReadFile(filepath.Join(f.extractDst, config.ZarfYAML)) - if err != nil { - return zarfTypes.ZarfPackage{}, err - } - var pkg zarfTypes.ZarfPackage - if err := goyaml.Unmarshal(p, &pkg); err != nil { - return zarfTypes.ZarfPackage{}, err - } - return pkg, err -} - // toBundle transfers a Zarf package to a given Bundle -func (f *localFetcher) toBundle(pkg zarfTypes.ZarfPackage, pkgTmp string) ([]ocispec.Descriptor, error) { - // todo: only grab components that are required + specified in optionalComponents +func (f *localFetcher) toBundle(pkgTmp string) ([]ocispec.Descriptor, error) { ctx := context.TODO() - src, err := file.New(pkgTmp) + + // load pkg and layout of pkg paths + pkgSrc := zarfSources.TarballSource{ + ZarfPackageOptions: &zarfTypes.ZarfPackageOptions{ + PackageSource: f.pkg.Path, + }, + } + pkg, pkgPaths, err := loadPkg(pkgTmp, &pkgSrc, f.pkg.OptionalComponents) if err != nil { return nil, err } - // Grab Zarf layers - var paths []string - err = filepath.Walk(pkgTmp, func(path string, info os.FileInfo, err error) error { - // Catch any errors that happened during the walk + + // get paths from pkgs to put in the bundle + var pathsToBundle []string + for _, fullPath := range pkgPaths.Files() { + pathsToBundle = append(pathsToBundle, fullPath) + } + + if len(f.pkg.OptionalComponents) > 0 { + + // read in images/index.json + var imgIndex ocispec.Index + if pkgPaths.Images.Index != "" { + indexBytes, err := os.ReadFile(pkgPaths.Images.Index) + if err != nil { + return nil, err + } + err = json.Unmarshal(indexBytes, &imgIndex) + if err != nil { + return nil, err + } + } + + // go into the pkg's image index and filter out optional components, grabbing img manifests of imgs to include + imgManifestsToInclude, err := boci.FilterImageIndex(pkg.Components, imgIndex) if err != nil { - return err + return nil, err } - // Add any resource that is not a directory to the paths of objects we will include into the package - if !info.IsDir() { - paths = append(paths, path) + // go through image index and get all images' config + layers + includeLayers, err := getImgLayerDigests(imgManifestsToInclude, pkgPaths) + if err != nil { + return nil, err } - return err - }) + + // filter paths to only include layers that are in includeLayers + filteredPaths := filterPkgPaths(pkgPaths, includeLayers, pkg.Components) + pathsToBundle = filteredPaths + } + + // create a file store in the same tmp dir as the Zarf pkg (used to create descs + layers) + src, err := file.New(pkgTmp) if err != nil { - return nil, fmt.Errorf("unable to get the layers in the package to publish: %w", err) + return nil, err } + // go through the paths that should be bundled and add them to the bundle store var descs []ocispec.Descriptor - for _, path := range paths { + for _, path := range pathsToBundle { name, err := filepath.Rel(pkgTmp, path) if err != nil { return nil, err } + // set media type to blob for all layers in the pkg mediaType := zoci.ZarfLayerMediaTypeBlob - // todo: try finding the desc with media type of image manifest, and rewrite it here! - // just iterate through it's layers and add the annotations to each layer, then push to the store and add to descs - - // adds title annotations to descs and creates layer to put in the store - // title annotations need to be added to the pkg root manifest - // Zarf image manifests already contain those title annotations in remote OCI repos, but they need to be added manually here - // if using a custom tmp dir that is not an absolute path, get working dir and prepend to path to make it absolute if !filepath.IsAbs(path) { wd, err := os.Getwd() @@ -184,6 +180,7 @@ func (f *localFetcher) toBundle(pkg zarfTypes.ZarfPackage, pkgTmp string) ([]oci path = filepath.Join(wd, path) } + // use the file store to create descs + layers that will be used to create the pkg root manifest desc, err := src.Add(ctx, name, mediaType, path) if err != nil { return nil, err @@ -193,25 +190,28 @@ func (f *localFetcher) toBundle(pkg zarfTypes.ZarfPackage, pkgTmp string) ([]oci return nil, err } - // push if layer doesn't already exist in bundleStore - // at this point, for some reason, many layers already exist in the store? + // push if layer to bundle store if it doesn't already exist if exists, err := f.cfg.Store.Exists(ctx, desc); !exists && err == nil { if err := f.cfg.Store.Push(ctx, desc, layer); err != nil { return nil, err } } + + // record descriptor for the pkg root manifest descs = append(descs, desc) } - // push the manifest config - // todo: I don't think this is making it to the local bundle - manifestConfigDesc, err := pushZarfManifestConfigFromMetadata(f.cfg.Store, &pkg.Metadata, &pkg.Build) + // create a pkg root manifest + config because it doesn't come with local Zarf pkgs + manifestConfigDesc, err := generatePkgManifestConfig(f.cfg.Store, &pkg.Metadata, &pkg.Build) if err != nil { return nil, err } - // push the manifest rootManifest, err := generatePkgManifest(f.cfg.Store, descs, manifestConfigDesc) - descs = append(descs, rootManifest) + if err != nil { + return nil, err + } + + descs = append(descs, rootManifest, manifestConfigDesc) // put digest in uds-bundle.yaml to reference during deploy f.cfg.Bundle.Packages[f.cfg.PkgIter].Ref = f.cfg.Bundle.Packages[f.cfg.PkgIter].Ref + "@" + rootManifest.Digest.String() @@ -221,7 +221,7 @@ func (f *localFetcher) toBundle(pkg zarfTypes.ZarfPackage, pkgTmp string) ([]oci return descs, err } -func pushZarfManifestConfigFromMetadata(store *ocistore.Store, metadata *zarfTypes.ZarfMetadata, build *zarfTypes.ZarfBuildData) (ocispec.Descriptor, error) { +func generatePkgManifestConfig(store *ocistore.Store, metadata *zarfTypes.ZarfMetadata, build *zarfTypes.ZarfBuildData) (ocispec.Descriptor, error) { annotations := map[string]string{ ocispec.AnnotationTitle: metadata.Name, ocispec.AnnotationDescription: metadata.Description, @@ -232,7 +232,7 @@ func pushZarfManifestConfigFromMetadata(store *ocistore.Store, metadata *zarfTyp Annotations: annotations, } - manifestConfigDesc, err := utils.ToOCIStore(manifestConfig, ocispec.MediaTypeImageManifest, store) + manifestConfigDesc, err := boci.ToOCIStore(manifestConfig, zoci.ZarfConfigMediaType, store) if err != nil { return ocispec.Descriptor{}, err } @@ -250,7 +250,7 @@ func generatePkgManifest(store *ocistore.Store, descs []ocispec.Descriptor, conf Layers: descs, } - manifestDesc, err := utils.ToOCIStore(manifest, zoci.ZarfLayerMediaTypeBlob, store) + manifestDesc, err := boci.ToOCIStore(manifest, zoci.ZarfLayerMediaTypeBlob, store) if err != nil { return ocispec.Descriptor{}, err } diff --git a/src/pkg/bundler/fetcher/remote.go b/src/pkg/bundler/fetcher/remote.go index 06f04a78..f8ac65c1 100644 --- a/src/pkg/bundler/fetcher/remote.go +++ b/src/pkg/bundler/fetcher/remote.go @@ -7,7 +7,6 @@ package fetcher import ( "context" "fmt" - "os" "path/filepath" "strings" @@ -16,6 +15,7 @@ import ( "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/cache" "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -23,6 +23,7 @@ import ( zarfTypes "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" + ocistore "oras.land/oras-go/v2/content/oci" ) // remoteFetcher fetches remote Zarf pkgs for local bundles @@ -38,138 +39,128 @@ func (f *remoteFetcher) Fetch() ([]ocispec.Descriptor, error) { fetchSpinner := message.NewProgressSpinner("Fetching package %s", f.pkg.Name) defer fetchSpinner.Stop() - layerDescs, err := f.layersToLocalBundle(fetchSpinner, f.cfg.PkgIter+1, f.cfg.NumPkgs) + // find layers in remote + fetchSpinner.Updatef("Fetching %s package layer metadata (package %d of %d)", f.pkg.Name, f.cfg.PkgIter+1, f.cfg.NumPkgs) + layersToCopy, err := boci.FindPkgLayers(*f.remote, f.pkgRootManifest, f.pkg.OptionalComponents) if err != nil { return nil, err } + fetchSpinner.Stop() - // grab layers for archiving - for _, layerDesc := range layerDescs { - if layerDesc.MediaType == ocispec.MediaTypeImageManifest { - // rewrite the Zarf image manifest to have media type of Zarf blob - err = os.Remove(filepath.Join(f.cfg.TmpDstDir, config.BlobsDir, layerDesc.Digest.Encoded())) - if err != nil { - return nil, err - } - err = utils.FetchLayerAndStore(layerDesc, f.remote.OrasRemote, f.cfg.Store) - if err != nil { - return nil, err - } - - // ensure media type is Zarf blob for layers in the bundle's root manifest - layerDesc.MediaType = zoci.ZarfLayerMediaTypeBlob - - // add layer to bundle's root manifest - f.cfg.BundleRootManifest.Layers = append(f.cfg.BundleRootManifest.Layers, layerDesc) - } - } - - fetchSpinner.Successf("Fetched package: %s", f.pkg.Name) - return layerDescs, nil -} - -// LayersToLocalBundle pushes a remote Zarf pkg's layers to a local bundle -func (f *remoteFetcher) layersToLocalBundle(spinner *message.Spinner, currentPackageIter int, totalPackages int) ([]ocispec.Descriptor, error) { - spinner.Updatef("Fetching %s package layer metadata (package %d of %d)", f.pkg.Name, currentPackageIter, totalPackages) - // get only the layers that are required by the components - layersToCopy, err := utils.GetZarfLayers(*f.remote, f.pkgRootManifest, f.pkg.OptionalComponents) - if err != nil { - return nil, err - } - spinner.Stop() - layerDescs, err := f.remoteToLocal(layersToCopy) + // copy layers to local bundle + fetchSpinner.Updatef("Pushing package %s layers to bundle (package %d of %d)", f.pkg.Name, f.cfg.PkgIter+1, f.cfg.NumPkgs) + pkgDescs, err := f.copyRemotePkgLayers(layersToCopy) if err != nil { return nil, err } - // return layer descriptor so we can copy them into the tarball path map - spinner.Updatef("Pushing package %s layers to registry (package %d of %d)", f.pkg.Name, currentPackageIter, totalPackages) - return layerDescs, err + + fetchSpinner.Successf("Fetched package: %s", f.pkg.Name) + return pkgDescs, nil } -// remoteToLocal copies a remote Zarf pkg to a local OCI store -func (f *remoteFetcher) remoteToLocal(layersToCopy []ocispec.Descriptor) ([]ocispec.Descriptor, error) { +// copyRemotePkgLayers copies a remote Zarf pkg to a local OCI store +func (f *remoteFetcher) copyRemotePkgLayers(layersToCopy []ocispec.Descriptor) ([]ocispec.Descriptor, error) { ctx := context.TODO() // pull layers from remote and write to OCI artifact dir var descsToBundle []ocispec.Descriptor var layersToPull []ocispec.Descriptor estimatedBytes := int64(0) + // grab descriptors of layers to copy for _, layer := range layersToCopy { if layer.Digest == "" { continue } - // check if layer already exists - if exists, _ := f.cfg.Store.Exists(ctx, layer); exists { - continue - } else if cache.Exists(layer.Digest.Encoded()) { - err := cache.Use(layer.Digest.Encoded(), filepath.Join(f.cfg.TmpDstDir, config.BlobsDir)) - if err != nil { - return nil, err - } - } else if layer.MediaType != ocispec.MediaTypeImageManifest { - // grab layer to pull from OCI; don't grab Zarf root manifest because we get it automatically during oras.Copy() + + exists, err := checkLayerExists(ctx, layer, f.cfg.Store, f.cfg.TmpDstDir) + if err != nil { + return nil, err + } + // if layers don't already exist on disk, add to layersToPull + // but don't grab Zarf root manifest (id'd by image manifest) because we get it automatically during oras.Copy() + if !exists && layer.MediaType != ocispec.MediaTypeImageManifest { layersToPull = append(layersToPull, layer) estimatedBytes += layer.Size } descsToBundle = append(descsToBundle, layer) } - // pull layers that didn't exist on disk + // pull layers that didn't already exist on disk if len(layersToPull) > 0 { - // copy Zarf pkg - copyOpts := utils.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) - // Create a thread to update a progress bar as we save the package to disk - doneSaving := make(chan error) - - // Grab tmpDirSize and add it to the estimatedBytes, otherwise the progress bar will be off - // because as multiple packages are pulled into the tmpDir, RenderProgressBarForLocalDirWrite continues to - // add their size which results in strange MB ratios - tmpDirSize, err := helpers.GetDirSize(f.cfg.TmpDstDir) - if err != nil { - return nil, err - } - - go zarfUtils.RenderProgressBarForLocalDirWrite(f.cfg.TmpDstDir, estimatedBytes+tmpDirSize, doneSaving, fmt.Sprintf("Pulling bundle: %s", f.pkg.Name), fmt.Sprintf("Successfully pulled package: %s", f.pkg.Name)) - rootPkgDesc, err := oras.Copy(context.TODO(), f.remote.Repo(), f.remote.Repo().Reference.String(), f.cfg.Store, "", copyOpts) - doneSaving <- err - <-doneSaving + rootPkgDesc, err := f.copyLayers(layersToPull, estimatedBytes) if err != nil { return nil, err } - // grab pkg root manifest for archiving + // grab pkg root manifest for archiving and save it to bundle root manifest descsToBundle = append(descsToBundle, rootPkgDesc) + rootPkgDesc.MediaType = zoci.ZarfLayerMediaTypeBlob // force media type to Zarf blob + f.cfg.BundleRootManifest.Layers = append(f.cfg.BundleRootManifest.Layers, rootPkgDesc) // cache only the image layers that were just pulled - for _, layer := range layersToPull { - if strings.Contains(layer.Annotations[ocispec.AnnotationTitle], config.BlobsDir) { - err = cache.Add(filepath.Join(f.cfg.TmpDstDir, config.BlobsDir, layer.Digest.Encoded())) - if err != nil { - return nil, err - } - } + err = cachePulledImgLayers(layersToPull, f.cfg.TmpDstDir) + if err != nil { + return nil, err } } else { - // need to grab pkg root manifest and config manually bc we didn't use oras.Copy() - pkgManifestDesc, err := utils.ToOCIStore(f.pkgRootManifest, ocispec.MediaTypeImageManifest, f.cfg.Store) + // no layers to pull but need to grab pkg root manifest and config manually bc we didn't use oras.Copy() + pkgManifestDesc, err := boci.ToOCIStore(f.pkgRootManifest, ocispec.MediaTypeImageManifest, f.cfg.Store) + if err != nil { + return nil, err + } + + // save pkg manifest to bundle root manifest + pkgManifestDesc.MediaType = zoci.ZarfLayerMediaTypeBlob // force media type to Zarf blob + f.cfg.BundleRootManifest.Layers = append(f.cfg.BundleRootManifest.Layers, pkgManifestDesc) + + manifestConfigDesc, err := boci.ToOCIStore(f.pkgRootManifest.Config, zoci.ZarfConfigMediaType, f.cfg.Store) if err != nil { return nil, err } - descsToBundle = append(descsToBundle, pkgManifestDesc) + descsToBundle = append(descsToBundle, pkgManifestDesc, manifestConfigDesc) } return descsToBundle, nil } +// copyLayers uses ORAS to copy layers from a remote repo to a local OCI store +func (f *remoteFetcher) copyLayers(layersToPull []ocispec.Descriptor, estimatedBytes int64) (ocispec.Descriptor, error) { + // copy Zarf pkg + copyOpts := boci.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) + // Create a thread to update a progress bar as we save the package to disk + doneSaving := make(chan error) + + // Grab tmpDirSize and add it to the estimatedBytes, otherwise the progress bar will be off + // because as multiple packages are pulled into the tmpDir, RenderProgressBarForLocalDirWrite continues to + // add their size which results in strange MB ratios + tmpDirSize, err := helpers.GetDirSize(f.cfg.TmpDstDir) + if err != nil { + return ocispec.Descriptor{}, err + } + + go zarfUtils.RenderProgressBarForLocalDirWrite(f.cfg.TmpDstDir, estimatedBytes+tmpDirSize, doneSaving, fmt.Sprintf("Pulling bundle: %s", f.pkg.Name), fmt.Sprintf("Successfully pulled package: %s", f.pkg.Name)) + rootPkgDesc, err := oras.Copy(context.TODO(), f.remote.Repo(), f.remote.Repo().Reference.String(), f.cfg.Store, "", copyOpts) + doneSaving <- err + <-doneSaving + if err != nil { + return ocispec.Descriptor{}, err + } + return rootPkgDesc, nil +} + func (f *remoteFetcher) GetPkgMetadata() (zarfTypes.ZarfPackage, error) { ctx := context.TODO() platform := ocispec.Platform{ Architecture: config.GetArch(), OS: oci.MultiOS, } + + // create OCI remote url := fmt.Sprintf("%s:%s", f.pkg.Repository, f.pkg.Ref) remote, err := zoci.NewRemote(url, platform) if err != nil { return zarfTypes.ZarfPackage{}, err } + + // get package metadata tmpDir, err := zarfUtils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { return zarfTypes.ZarfPackage{}, fmt.Errorf("bundler unable to create temp directory: %w", err) @@ -177,6 +168,8 @@ func (f *remoteFetcher) GetPkgMetadata() (zarfTypes.ZarfPackage, error) { if _, err := remote.PullPackageMetadata(ctx, tmpDir); err != nil { return zarfTypes.ZarfPackage{}, err } + + // read metadata zarfYAML := zarfTypes.ZarfPackage{} zarfYAMLPath := filepath.Join(tmpDir, config.ZarfYAML) err = utils.ReadYAMLStrict(zarfYAMLPath, &zarfYAML) @@ -185,3 +178,30 @@ func (f *remoteFetcher) GetPkgMetadata() (zarfTypes.ZarfPackage, error) { } return zarfYAML, err } + +// cachePulledImgLayers caches the image layers that were just pulled +func cachePulledImgLayers(pulledLayers []ocispec.Descriptor, dstDir string) (err error) { + for _, layer := range pulledLayers { + if strings.Contains(layer.Annotations[ocispec.AnnotationTitle], config.BlobsDir) { + err = cache.Add(filepath.Join(dstDir, config.BlobsDir, layer.Digest.Encoded())) + if err != nil { + return err + } + } + } + return nil +} + +// checkLayerExists checks if a layer already exists in the bundle store or the cache +func checkLayerExists(ctx context.Context, layer ocispec.Descriptor, store *ocistore.Store, dstDir string) (bool, error) { + if exists, _ := store.Exists(ctx, layer); exists { + return true, nil + } else if cache.Exists(layer.Digest.Encoded()) { + err := cache.Use(layer.Digest.Encoded(), filepath.Join(dstDir, config.BlobsDir)) + if err != nil { + return false, err + } + return true, nil + } + return false, nil +} diff --git a/src/pkg/bundler/localbundle.go b/src/pkg/bundler/localbundle.go index dfb61519..05b4091d 100644 --- a/src/pkg/bundler/localbundle.go +++ b/src/pkg/bundler/localbundle.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/bundler/fetcher" "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/zoci" @@ -92,13 +93,13 @@ func (lo *LocalBundle) create(signature []byte) error { if err != nil { return err } - layerDescs, err := pkgFetcher.Fetch() + pkgDescs, err := pkgFetcher.Fetch() if err != nil { return err } - // add to artifactPathMap for local tarball - // todo: if we know the path to where the blobs are stored, we can use that instead of the artifactPathMap? - for _, layer := range layerDescs { + + // add to artifactPathMap for local bundle tarball + for _, layer := range pkgDescs { digest := layer.Digest.Encoded() artifactPathMap[filepath.Join(lo.tmpDstDir, config.BlobsDir, digest)] = filepath.Join(config.BlobsDir, digest) } @@ -128,7 +129,7 @@ func (lo *LocalBundle) create(signature []byte) error { rootManifest.Config = manifestConfigDesc rootManifest.SchemaVersion = 2 rootManifest.Annotations = manifestAnnotationsFromMetadata(&bundle.Metadata) // maps to registry UI - rootManifestDesc, err := utils.ToOCIStore(rootManifest, ocispec.MediaTypeImageManifest, store) + rootManifestDesc, err := boci.ToOCIStore(rootManifest, ocispec.MediaTypeImageManifest, store) if err != nil { return err } @@ -205,7 +206,7 @@ func pushManifestConfig(store *ocistore.Store, metadata types.UDSMetadata, build OCIVersion: "1.0.1", Annotations: annotations, } - manifestConfigDesc, err := utils.ToOCIStore(manifestConfig, zoci.ZarfLayerMediaTypeBlob, store) + manifestConfigDesc, err := boci.ToOCIStore(manifestConfig, zoci.ZarfLayerMediaTypeBlob, store) if err != nil { return ocispec.Descriptor{}, err } diff --git a/src/pkg/bundler/pusher/remote.go b/src/pkg/bundler/pusher/remote.go index 6a058099..a9a73cd2 100644 --- a/src/pkg/bundler/pusher/remote.go +++ b/src/pkg/bundler/pusher/remote.go @@ -11,7 +11,7 @@ import ( "github.com/defenseunicorns/pkg/oci" "github.com/defenseunicorns/uds-cli/src/config" - "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/zoci" @@ -63,10 +63,10 @@ func (p *RemotePusher) Push() (ocispec.Descriptor, error) { return zarfManifestDesc, nil } -// PushManifest pushes the Zarf pkg's manifest to either a local or remote bundle +// PushManifest pushes the Zarf pkg's manifest to a remote bundle func (p *RemotePusher) PushManifest() (ocispec.Descriptor, error) { var zarfManifestDesc ocispec.Descriptor - desc, err := utils.ToOCIRemote(p.cfg.PkgRootManifest, zoci.ZarfLayerMediaTypeBlob, p.cfg.RemoteDst.OrasRemote) + desc, err := boci.ToOCIRemote(p.cfg.PkgRootManifest, zoci.ZarfLayerMediaTypeBlob, p.cfg.RemoteDst.OrasRemote) if err != nil { return ocispec.Descriptor{}, err } @@ -78,7 +78,7 @@ func (p *RemotePusher) PushManifest() (ocispec.Descriptor, error) { func (p *RemotePusher) LayersToRemoteBundle(spinner *message.Spinner, currentPackageIter int, totalPackages int) ([]ocispec.Descriptor, error) { spinner.Updatef("Fetching %s package layer metadata (package %d of %d)", p.pkg.Name, currentPackageIter, totalPackages) // get only the layers that are required by the components - layersToCopy, err := utils.GetZarfLayers(p.cfg.RemoteSrc, p.cfg.PkgRootManifest, p.pkg.OptionalComponents) + layersToCopy, err := boci.FindPkgLayers(p.cfg.RemoteSrc, p.cfg.PkgRootManifest, p.pkg.OptionalComponents) if err != nil { return nil, err } diff --git a/src/pkg/bundler/remotebundle.go b/src/pkg/bundler/remotebundle.go index 5bce9950..db4d118a 100644 --- a/src/pkg/bundler/remotebundle.go +++ b/src/pkg/bundler/remotebundle.go @@ -11,7 +11,7 @@ import ( "github.com/defenseunicorns/pkg/oci" "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/bundler/pusher" - "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/zoci" @@ -47,7 +47,7 @@ func (r *RemoteBundle) create(signature []byte) error { ctx := context.TODO() // set the bundle remote's reference from metadata - r.output = utils.EnsureOCIPrefix(r.output) + r.output = boci.EnsureOCIPrefix(r.output) ref, err := referenceFromMetadata(r.output, &r.bundle.Metadata) if err != nil { return err @@ -137,7 +137,7 @@ func (r *RemoteBundle) create(signature []byte) error { message.Debug("Pushed config:", message.JSONValue(configDesc)) // check for existing index - index, err := utils.GetIndex(bundleRemote.OrasRemote, dstRef.String()) + index, err := boci.GetIndex(bundleRemote.OrasRemote, dstRef.String()) if err != nil { return err } @@ -146,13 +146,13 @@ func (r *RemoteBundle) create(signature []byte) error { rootManifest.Config = configDesc rootManifest.SchemaVersion = 2 rootManifest.Annotations = manifestAnnotationsFromMetadata(&bundle.Metadata) // maps to registry UI - rootManifestDesc, err := utils.ToOCIRemote(rootManifest, ocispec.MediaTypeImageManifest, bundleRemote.OrasRemote) + rootManifestDesc, err := boci.ToOCIRemote(rootManifest, ocispec.MediaTypeImageManifest, bundleRemote.OrasRemote) if err != nil { return err } // create or update, then push index.json - err = utils.UpdateIndex(index, bundleRemote.OrasRemote, bundle, *rootManifestDesc) + err = boci.UpdateIndex(index, bundleRemote.OrasRemote, bundle, *rootManifestDesc) if err != nil { return err } diff --git a/src/pkg/sources/new.go b/src/pkg/sources/new.go index 6149cf9a..8c902772 100644 --- a/src/pkg/sources/new.go +++ b/src/pkg/sources/new.go @@ -9,6 +9,7 @@ import ( "github.com/defenseunicorns/pkg/oci" "github.com/defenseunicorns/uds-cli/src/config" + "github.com/defenseunicorns/uds-cli/src/types" zarfSources "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/zoci" zarfTypes "github.com/defenseunicorns/zarf/src/types" @@ -16,11 +17,11 @@ import ( ) // New creates a new package source based on pkgLocation -func New(pkgLocation string, pkgName string, opts zarfTypes.ZarfPackageOptions, sha string, nsOverrides NamespaceOverrideMap) (zarfSources.PackageSource, error) { +func New(pkgLocation string, pkg types.Package, opts zarfTypes.ZarfPackageOptions, sha string, nsOverrides NamespaceOverrideMap) (zarfSources.PackageSource, error) { var source zarfSources.PackageSource if strings.Contains(pkgLocation, "tar.zst") { source = &TarballBundle{ - PkgName: pkgName, + Pkg: pkg, PkgOpts: &opts, PkgManifestSHA: sha, TmpDir: opts.PackageSource, @@ -37,7 +38,7 @@ func New(pkgLocation string, pkgName string, opts zarfTypes.ZarfPackageOptions, return nil, err } source = &RemoteBundle{ - PkgName: pkgName, + Pkg: pkg, PkgOpts: &opts, PkgManifestSHA: sha, TmpDir: opts.PackageSource, diff --git a/src/pkg/sources/remote.go b/src/pkg/sources/remote.go index 0ed64f6c..825bc616 100644 --- a/src/pkg/sources/remote.go +++ b/src/pkg/sources/remote.go @@ -15,8 +15,9 @@ import ( "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/cache" "github.com/defenseunicorns/uds-cli/src/pkg/utils" + "github.com/defenseunicorns/uds-cli/src/pkg/utils/boci" + "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -30,12 +31,11 @@ import ( // RemoteBundle is a package source for remote bundles that implements Zarf's packager.PackageSource type RemoteBundle struct { - PkgName string + Pkg types.Package PkgOpts *zarfTypes.ZarfPackageOptions PkgManifestSHA string TmpDir string Remote *oci.OrasRemote - isPartial bool nsOverrides NamespaceOverrideMap } @@ -64,7 +64,9 @@ func (r *RemoteBundle) LoadPackage(dst *layout.PackagePaths, filter filters.Comp dst.SetFromLayers(layers) - err = sources.ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, r.isPartial) + isPartialPkg := r.PkgOpts.OptionalComponents != "" + + err = sources.ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, isPartialPkg) if err != nil { return zarfTypes.ZarfPackage{}, nil, err } @@ -96,7 +98,7 @@ func (r *RemoteBundle) LoadPackage(dst *layout.PackagePaths, filter filters.Comp } // ensure we're using the correct package name as specified by the bundle - pkg.Metadata.Name = r.PkgName + pkg.Metadata.Name = r.Pkg.Name return pkg, nil, err } @@ -109,7 +111,7 @@ func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ b } pkgManifestDesc := root.Locate(r.PkgManifestSHA) if oci.IsEmptyDescriptor(pkgManifestDesc) { - return zarfTypes.ZarfPackage{}, nil, fmt.Errorf("zarf package %s with manifest sha %s not found", r.PkgName, r.PkgManifestSHA) + return zarfTypes.ZarfPackage{}, nil, fmt.Errorf("zarf package %s with manifest sha %s not found", r.Pkg.Name, r.PkgManifestSHA) } // look at Zarf pkg manifest, grab zarf.yaml desc and download it @@ -159,7 +161,7 @@ func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ b err = sources.ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, true) // ensure we're using the correct package name as specified by the bundle - pkg.Metadata.Name = r.PkgName + pkg.Metadata.Name = r.Pkg.Name return pkg, nil, err } @@ -187,36 +189,44 @@ func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, erro return nil, err } - // only fetch layers that exist in the remote as optional ones might not exist - // todo: this is incredibly slow; maybe keep track of layers in bundle metadata instead of having to query the remote? - progressBar := message.NewProgressBar(int64(len(pkgManifest.Layers)), fmt.Sprintf("Verifying layers in Zarf package: %s", r.PkgName)) estimatedBytes := int64(0) layersToPull := []ocispec.Descriptor{pkgManifestDesc} layersInBundle := []ocispec.Descriptor{pkgManifestDesc} - for _, layer := range pkgManifest.Layers { - ok, err := r.Remote.Repo().Blobs().Exists(ctx, layer) - if err != nil { - return nil, err - } - progressBar.Add(1) - if ok { - estimatedBytes += layer.Size - layersInBundle = append(layersInBundle, layer) - digest := layer.Digest.Encoded() - if strings.Contains(layer.Annotations[ocispec.AnnotationTitle], config.BlobsDir) && cache.Exists(digest) { - dst := filepath.Join(r.TmpDir, "images", config.BlobsDir) - err = cache.Use(digest, dst) - if err != nil { - return nil, err + // get pkg layers that we want to pull + pkgLayers, _, err := boci.FindBundledPkgLayers(ctx, r.Pkg, rootManifest, r.Remote) + if err != nil { + return nil, err + } + + // todo: we seem to need to specifically pull the layers from the pkgManifest here, but not in the + // other location that FindBundledPkgLayers is called. Why is that? + // I believe it's bc here we are going to iterate through those layers and fill out a layout with + // the annotations from each desc (only pkgManifest layers contain the necessary annotations) + + // correlate descs in pkg root manifest with the pkg layers to pull + for _, manifestLayer := range pkgManifest.Layers { + for _, pkgLayer := range pkgLayers { + if pkgLayer.Digest.Encoded() == manifestLayer.Digest.Encoded() { + layersInBundle = append(layersInBundle, manifestLayer) + digest := manifestLayer.Digest.Encoded() + + // if it's an image layer and is in the cache, use it + if strings.Contains(manifestLayer.Annotations[ocispec.AnnotationTitle], config.BlobsDir) && cache.Exists(digest) { + dst := filepath.Join(r.TmpDir, "images", config.BlobsDir) + err = cache.Use(digest, dst) + if err != nil { + return nil, err + } + } else { + // not in cache, so pull + layersToPull = append(layersToPull, manifestLayer) + estimatedBytes += manifestLayer.Size } - } else { - layersToPull = append(layersToPull, layer) + break // if layer is found, break out of inner loop } - } } - progressBar.Successf("Verified %s package", r.PkgName) store, err := file.New(r.TmpDir) if err != nil { @@ -225,9 +235,9 @@ func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, erro defer store.Close() // copy zarf pkg to local store - copyOpts := utils.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) + copyOpts := boci.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) doneSaving := make(chan error) - go zarfUtils.RenderProgressBarForLocalDirWrite(r.TmpDir, estimatedBytes, doneSaving, fmt.Sprintf("Pulling bundled Zarf pkg: %s", r.PkgName), fmt.Sprintf("Successfully pulled package: %s", r.PkgName)) + go zarfUtils.RenderProgressBarForLocalDirWrite(r.TmpDir, estimatedBytes, doneSaving, fmt.Sprintf("Pulling bundled Zarf pkg: %s", r.Pkg.Name), fmt.Sprintf("Successfully pulled package: %s", r.Pkg.Name)) _, err = oras.Copy(ctx, r.Remote.Repo(), r.Remote.Repo().Reference.String(), store, "", copyOpts) doneSaving <- err @@ -236,9 +246,5 @@ func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, erro return nil, err } - // need to substract 1 from layersInBundle because it includes the pkgManifestDesc and pkgManifest.Layers does not - if len(pkgManifest.Layers) != len(layersInBundle)-1 { - r.isPartial = true - } return layersInBundle, nil } diff --git a/src/pkg/sources/tarball.go b/src/pkg/sources/tarball.go index 62c114fc..0917aa5a 100644 --- a/src/pkg/sources/tarball.go +++ b/src/pkg/sources/tarball.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/pkg/oci" + "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/filters" @@ -35,15 +36,14 @@ type TarballBundle struct { PkgManifestSHA string TmpDir string BundleLocation string - PkgName string - isPartial bool + Pkg types.Package nsOverrides NamespaceOverrideMap } // LoadPackage loads a Zarf package from a local tarball bundle func (t *TarballBundle) LoadPackage(dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) (zarfTypes.ZarfPackage, []string, error) { - packageSpinner := message.NewProgressSpinner("Loading bundled Zarf package: %s", t.PkgName) + packageSpinner := message.NewProgressSpinner("Loading bundled Zarf package: %s", t.Pkg.Name) defer packageSpinner.Stop() files, err := t.extractPkgFromBundle() @@ -68,7 +68,9 @@ func (t *TarballBundle) LoadPackage(dst *layout.PackagePaths, filter filters.Com dst.SetFromPaths(files) - if err := sources.ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, t.isPartial); err != nil { + isPartialPkg := t.PkgOpts.OptionalComponents != "" + + if err := sources.ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, isPartialPkg); err != nil { return zarfTypes.ZarfPackage{}, nil, err } @@ -98,9 +100,9 @@ func (t *TarballBundle) LoadPackage(dst *layout.PackagePaths, filter filters.Com setAsYOLO(&pkg) } - packageSpinner.Successf("Loaded bundled Zarf package: %s", t.PkgName) + packageSpinner.Successf("Loaded bundled Zarf package: %s", t.Pkg.Name) // ensure we're using the correct package name as specified by the bundle - pkg.Metadata.Name = t.PkgName + pkg.Metadata.Name = t.Pkg.Name return pkg, nil, err } @@ -195,7 +197,7 @@ func (t *TarballBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ err = sourceArchive.Close() // ensure we're using the correct package name as specified by the bundle - pkg.Metadata.Name = t.PkgName + pkg.Metadata.Name = t.Pkg.Name return pkg, nil, err } @@ -284,8 +286,5 @@ func (t *TarballBundle) extractPkgFromBundle() ([]string, error) { } defer sourceArchive.Close() err = format.Extract(context.TODO(), sourceArchive, layersToExtract, extractLayer) - if len(manifest.Layers) > len(files) { - t.isPartial = true - } return files, err } diff --git a/src/pkg/utils/oci.go b/src/pkg/utils/boci/oci.go similarity index 62% rename from src/pkg/utils/oci.go rename to src/pkg/utils/boci/oci.go index 53a67b17..be00e717 100644 --- a/src/pkg/utils/oci.go +++ b/src/pkg/utils/boci/oci.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2023-Present The UDS Authors -// Package utils provides utility fns for UDS-CLI -package utils +// Package boci (bundle OCI) provides OCI utility functions for bundles +package boci import ( "bytes" @@ -10,15 +10,19 @@ import ( "encoding/json" "errors" "fmt" + "io" "slices" "strings" "github.com/defenseunicorns/pkg/oci" "github.com/defenseunicorns/uds-cli/src/config" + "github.com/defenseunicorns/uds-cli/src/pkg/utils" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/zoci" zarfTypes "github.com/defenseunicorns/zarf/src/types" + goyaml "github.com/goccy/go-yaml" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" @@ -26,18 +30,6 @@ import ( "oras.land/oras-go/v2/errdef" ) -// FetchLayerAndStore fetches a remote layer and copies it to a local store -func FetchLayerAndStore(layerDesc ocispec.Descriptor, remoteRepo *oci.OrasRemote, localStore *ocistore.Store) error { - ctx := context.TODO() - layerBytes, err := remoteRepo.FetchLayer(ctx, layerDesc) - if err != nil { - return err - } - rootPkgDescBytes := content.NewDescriptorFromBytes(zoci.ZarfLayerMediaTypeBlob, layerBytes) - err = localStore.Push(context.TODO(), rootPkgDescBytes, bytes.NewReader(layerBytes)) - return err -} - // ToOCIStore takes an arbitrary type, typically a struct, marshals it into JSON and store it in a local OCI store func ToOCIStore(t any, mediaType string, store *ocistore.Store) (ocispec.Descriptor, error) { b, err := json.Marshal(t) @@ -258,15 +250,15 @@ func EnsureOCIPrefix(source string) string { return source } -// GetZarfLayers grabs the necessary Zarf pkg layers from a remote OCI registry -func GetZarfLayers(remote zoci.Remote, pkgRootManifest *oci.Manifest, optionalComponents []string) ([]ocispec.Descriptor, error) { +// FindPkgLayers finds the necessary Zarf pkg layers from a remote OCI registry +func FindPkgLayers(remote zoci.Remote, pkgRootManifest *oci.Manifest, optionalComponents []string) ([]ocispec.Descriptor, error) { ctx := context.TODO() zarfPkg, err := remote.FetchZarfYAML(ctx) if err != nil { return nil, err } - // ensure we're only pulling required components and optional components + // ensure we're only pulling required components and optional components and images var components []zarfTypes.ZarfComponent for _, c := range zarfPkg.Components { if c.Required != nil || slices.Contains(optionalComponents, c.Name) { @@ -290,3 +282,159 @@ func GetZarfLayers(remote zoci.Remote, pkgRootManifest *oci.Manifest, optionalCo layersToCopy = append(layersToCopy, pkgRootManifest.Config) return layersToCopy, err } + +// FilterImageIndex filters out optional components from the images index +func FilterImageIndex(components []zarfTypes.ZarfComponent, imgIndex ocispec.Index) ([]ocispec.Descriptor, error) { + // include only images that are in the components using a map to dedup manifests + manifestIncludeMap := map[string]ocispec.Descriptor{} + for _, manifest := range imgIndex.Manifests { + for _, component := range components { + for _, imgName := range component.Images { + // include backwards compatibility shim for older Zarf versions that would leave docker.io off of image annotations + if manifest.Annotations[ocispec.AnnotationBaseImageName] == imgName || + manifest.Annotations[ocispec.AnnotationBaseImageName] == fmt.Sprintf("docker.io/%s", imgName) { + manifestIncludeMap[manifest.Digest.Hex()] = manifest + } + } + } + } + // convert map to list and rewrite the index manifests + var manifestsToInclude []ocispec.Descriptor + for _, manifest := range manifestIncludeMap { + manifestsToInclude = append(manifestsToInclude, manifest) + } + + return manifestsToInclude, nil +} + +// FindBundledPkgLayers finds the necessary Zarf pkg layers from a remote bundle +func FindBundledPkgLayers(ctx context.Context, pkg types.Package, rootManifest *oci.Manifest, remote *oci.OrasRemote) ([]ocispec.Descriptor, int64, error) { + var layersToPull []ocispec.Descriptor + estPkgBytes := int64(0) + + // grab sha of zarf image manifest and pull it down + sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle + manifestDesc := rootManifest.Locate(sha) + manifestBytes, err := remote.FetchLayer(ctx, manifestDesc) + if err != nil { + return nil, 0, err + } + + // unmarshal the zarf image manifest and add it to the layers to pull + var manifest oci.Manifest + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return nil, 0, err + } + + layersToPull = append(layersToPull, manifestDesc) + + filteredComponents, err := getFilteredComponents(ctx, remote, manifest, pkg.OptionalComponents) + if err != nil { + return nil, 0, err + } + + // go through manifest layers and add to layersToPull as appropriate + var imgIndex ocispec.Index + for _, desc := range manifest.Layers { + descAnnotationTile := desc.Annotations[ocispec.AnnotationTitle] + if descAnnotationTile == "images/index.json" { + imgIndex, err = handleImgIndex(ctx, remote, desc) + if err != nil { + return nil, 0, err + } + layersToPull = append(layersToPull, desc) + } else if strings.HasPrefix(descAnnotationTile, "components/") { + if shouldInclude := utils.IncludeComponent(descAnnotationTile, filteredComponents); shouldInclude { + layersToPull = append(layersToPull, desc) + } + } else if !strings.Contains(descAnnotationTile, config.BlobsDir) { + // not a blob or component, add to layersToPull + layersToPull = append(layersToPull, desc) + } + } + + // get only image manifests that are part of req'd + selected components + manifestsToInclude, err := FilterImageIndex(filteredComponents, imgIndex) + if err != nil { + return nil, 0, err + } + + // grab all layers from the included image manifests + for _, desc := range manifestsToInclude { + imgManifest, err := getImgManifest(ctx, remote, desc) + if err != nil { + return nil, 0, err + } + + // grab all layers in image manifest, the img config and the img manifest itself + layersToPull = append(layersToPull, imgManifest.Layers...) + layersToPull = append(layersToPull, desc, imgManifest.Config) + } + + // loop through layersToPull and add up bytes + for _, layer := range layersToPull { + estPkgBytes += layer.Size + } + + return layersToPull, estPkgBytes, nil +} + +func getImgManifest(ctx context.Context, remote *oci.OrasRemote, desc ocispec.Descriptor) (ocispec.Manifest, error) { + imgManifestReader, err := remote.Repo().Blobs().Fetch(ctx, desc) + if err != nil { + return ocispec.Manifest{}, err + } + imgManifestBytes, err := io.ReadAll(imgManifestReader) + if err != nil { + return ocispec.Manifest{}, err + } + var imgManifest ocispec.Manifest + if err := json.Unmarshal(imgManifestBytes, &imgManifest); err != nil { + return ocispec.Manifest{}, err + } + err = imgManifestReader.Close() + if err != nil { + return ocispec.Manifest{}, err + } + return imgManifest, nil +} + +func handleImgIndex(ctx context.Context, remote *oci.OrasRemote, desc ocispec.Descriptor) (ocispec.Index, error) { + indexBytes, err := remote.FetchLayer(ctx, desc) + if err != nil { + return ocispec.Index{}, err + } + + var index ocispec.Index + if err := json.Unmarshal(indexBytes, &index); err != nil { + return ocispec.Index{}, err + } + return index, nil +} + +func getFilteredComponents(ctx context.Context, remote *oci.OrasRemote, manifest oci.Manifest, optionalComponents []string) ([]zarfTypes.ZarfComponent, error) { + // get Zarf pkg from manifest + var zarfPkg zarfTypes.ZarfPackage + for _, desc := range manifest.Layers { + if desc.Annotations[ocispec.AnnotationTitle] == config.ZarfYAML { + zarfYAMLBytes, err := remote.FetchLayer(ctx, desc) + if err != nil { + return nil, err + } + if err := goyaml.Unmarshal(zarfYAMLBytes, &zarfPkg); err != nil { + return nil, err + } + break + } + } + + // create filter for optional components and filter the pkg + createFilter := filters.Combine( + filters.ForDeploy(strings.Join(optionalComponents, ","), false), + ) + filteredComponents, err := createFilter.Apply(zarfPkg) + if err != nil { + return nil, err + } + return filteredComponents, nil +} diff --git a/src/pkg/utils/utils.go b/src/pkg/utils/utils.go index f3b0b5fc..05b12b25 100644 --- a/src/pkg/utils/utils.go +++ b/src/pkg/utils/utils.go @@ -16,6 +16,7 @@ import ( "strings" "time" + zarfTypes "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" "github.com/defenseunicorns/pkg/helpers" @@ -43,6 +44,19 @@ func IsValidTarballPath(path string) bool { return re.MatchString(name) } +// IncludeComponent checks if a component has been specified in a a list of components (used for filtering optional components) +func IncludeComponent(componentToCheck string, filteredComponents []zarfTypes.ZarfComponent) bool { + for _, component := range filteredComponents { + // get component name from annotation + nameWithSuffix := strings.Split(componentToCheck, "components/")[1] + componentName := strings.Split(nameWithSuffix, ".tar")[0] + if componentName == component.Name { + return true + } + } + return false +} + // ConfigureLogs sets up the log file, log cache and output for the CLI func ConfigureLogs(cmd *cobra.Command) error { // don't configure UDS logs for vendored cmds diff --git a/src/test/bundles/13-composable-component/uds-bundle.yaml b/src/test/bundles/13-composable-component/uds-bundle.yaml index a4182f81..d5ac92d6 100644 --- a/src/test/bundles/13-composable-component/uds-bundle.yaml +++ b/src/test/bundles/13-composable-component/uds-bundle.yaml @@ -12,3 +12,5 @@ packages: # remote pkg ensures we're testing pulling composed components from OCI repository: localhost:888/prometheus ref: 0.0.1 + optionalComponents: + - upload-image diff --git a/src/test/bundles/14-optional-components/uds-bundle.yaml b/src/test/bundles/14-optional-components/uds-bundle.yaml new file mode 100644 index 00000000..e9486981 --- /dev/null +++ b/src/test/bundles/14-optional-components/uds-bundle.yaml @@ -0,0 +1,18 @@ +kind: UDSBundle +metadata: + name: optional-components + description: test bundle with optional components in its pkgs + version: 0.0.1 + +packages: + - name: prometheus + repository: localhost:888/prometheus + ref: 0.0.1 + optionalComponents: + - upload-image + + - name: podinfo-and-nginx + path: ../../packages/podinfo-and-nginx + ref: 0.0.1 + optionalComponents: + - podinfo diff --git a/src/test/common.go b/src/test/common.go index d2018597..b6e5475e 100644 --- a/src/test/common.go +++ b/src/test/common.go @@ -91,8 +91,14 @@ func (e2e *UDSE2ETest) GetLogFileContents(t *testing.T, stdErr string) string { // SetupDockerRegistry uses the host machine's docker daemon to spin up a local registry for testing purposes. func (e2e *UDSE2ETest) SetupDockerRegistry(t *testing.T, port int) { + // check if registry is already running on port + _, _, err := exec.Cmd("docker", "inspect", fmt.Sprintf("registry-%d", port)) + if err == nil { + fmt.Println("Registry already running, skipping setup") + return + } registryImage := "registry:2.8.3" - err := exec.CmdWithPrint("docker", "run", "-d", "--restart=always", "-p", fmt.Sprintf("%d:5000", port), "--name", fmt.Sprintf("registry-%d", port), registryImage) + err = exec.CmdWithPrint("docker", "run", "-d", "--restart=always", "-p", fmt.Sprintf("%d:5000", port), "--name", fmt.Sprintf("registry-%d", port), registryImage) require.NoError(t, err) // Check for registry health @@ -159,7 +165,7 @@ func (e2e *UDSE2ETest) DownloadZarfInitPkg(t *testing.T, zarfVersion string) { require.NoError(t, err) } -// CreateZarfPkg creates a Zarf package in the given path (todo: makefile?) +// CreateZarfPkg creates a Zarf package in the given path func (e2e *UDSE2ETest) CreateZarfPkg(t *testing.T, path string, forceCreate bool) { // check if pkg already exists pattern := fmt.Sprintf("%s/*-%s-*.tar.zst", path, e2e.Arch) diff --git a/src/test/e2e/bundle_test.go b/src/test/e2e/bundle_test.go index 06aa297a..6274d81b 100644 --- a/src/test/e2e/bundle_test.go +++ b/src/test/e2e/bundle_test.go @@ -85,9 +85,10 @@ func TestBundleWithLocalAndRemotePkgs(t *testing.T) { e2e.CreateZarfPkg(t, "src/test/packages/podinfo", false) bundleDir := "src/test/bundles/03-local-and-remote" - bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-test-local-and-remote-%s-0.0.1.tar.zst", e2e.Arch)) + bundleTarballName := fmt.Sprintf("uds-bundle-test-local-and-remote-%s-0.0.1.tar.zst", e2e.Arch) + bundlePath := filepath.Join(bundleDir, bundleTarballName) + pulledBundlePath := filepath.Join("build", bundleTarballName) - tarballPath := filepath.Join("build", fmt.Sprintf("uds-bundle-test-local-and-remote-%s-0.0.1.tar.zst", e2e.Arch)) bundleRef := registry.Reference{ Registry: "oci://localhost:888", // this info is derived from the bundle's metadata @@ -101,15 +102,15 @@ func TestBundleWithLocalAndRemotePkgs(t *testing.T) { t.Run("Test pulling and deploying from the same registry", func(t *testing.T) { publishInsecure(t, bundlePath, bundleRef.Registry) - pull(t, bundleRef.String(), tarballPath) // note that pull pulls the bundle into the build dir - deploy(t, filepath.Join("build", filepath.Base(bundlePath))) - remove(t, filepath.Join("build", filepath.Base(bundlePath))) + pull(t, bundleRef.String(), bundleTarballName) // note that pull pulls the bundle into the build dir + deploy(t, pulledBundlePath) + remove(t, pulledBundlePath) }) t.Run(" Test publishing and deploying from different registries", func(t *testing.T) { publishInsecure(t, bundlePath, bundleRef.Registry) - pull(t, bundleRef.String(), tarballPath) // note that pull pulls the bundle into the build dir - publishInsecure(t, filepath.Join("build", filepath.Base(bundlePath)), "oci://localhost:889") + pull(t, bundleRef.String(), bundleTarballName) // note that pull pulls the bundle into the build dir + publishInsecure(t, pulledBundlePath, "oci://localhost:889") deployInsecure(t, bundleRef.String()) }) } @@ -274,14 +275,15 @@ func TestRemoteBundleWithRemotePkgs(t *testing.T) { Reference: "0.0.1", } - tarballPath := filepath.Join("build", fmt.Sprintf("uds-bundle-example-remote-%s-0.0.1.tar.zst", e2e.Arch)) + bundleTarballName := fmt.Sprintf("uds-bundle-example-remote-%s-0.0.1.tar.zst", e2e.Arch) + tarballPath := filepath.Join("build", bundleTarballName) bundlePath := "src/test/bundles/01-uds-bundle" createRemoteInsecure(t, bundlePath, bundleRef.Registry, e2e.Arch) // Test without oci prefix createRemoteInsecure(t, bundlePath, "localhost:888", e2e.Arch) - pull(t, bundleRef.String(), tarballPath) + pull(t, bundleRef.String(), bundleTarballName) inspectRemoteInsecure(t, bundleRef.String()) inspectRemoteAndSBOMExtract(t, bundleRef.String()) deployAndRemoveLocalAndRemoteInsecure(t, bundleRef.String(), tarballPath) @@ -391,8 +393,9 @@ func TestPackageNaming(t *testing.T) { zarfPublish(t, pkg, "localhost:889") bundleDir := "src/test/bundles/10-package-naming" - bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-package-naming-%s-0.0.1.tar.zst", e2e.Arch)) - tarballPath := filepath.Join("build", fmt.Sprintf("uds-bundle-package-naming-%s-0.0.1.tar.zst", e2e.Arch)) + bundleTarballName := fmt.Sprintf("uds-bundle-package-naming-%s-0.0.1.tar.zst", e2e.Arch) + bundlePath := filepath.Join(bundleDir, bundleTarballName) + tarballPath := filepath.Join("build", bundleTarballName) bundleRef := registry.Reference{ Registry: "oci://localhost:888", // this info is derived from the bundle's metadata @@ -401,7 +404,7 @@ func TestPackageNaming(t *testing.T) { } createLocal(t, bundleDir, e2e.Arch) // todo: allow creating from both the folder containing and direct reference to uds-bundle.yaml publishInsecure(t, bundlePath, bundleRef.Registry) - pull(t, bundleRef.String(), tarballPath) + pull(t, bundleRef.String(), bundleTarballName) deploy(t, tarballPath) remove(t, tarballPath) @@ -411,14 +414,16 @@ func TestPackageNaming(t *testing.T) { } func TestBundleIndexInRemoteOnPublish(t *testing.T) { + deployZarfInit(t) e2e.SetupDockerRegistry(t, 888) defer e2e.TeardownRegistry(t, 888) bundleDir := "src/test/bundles/06-ghcr" bundleName := "ghcr-test" + bundleTarballName := fmt.Sprintf("uds-bundle-%s-%s-0.0.1.tar.zst", bundleName, e2e.Arch) bundlePathARM := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-%s-%s-0.0.1.tar.zst", bundleName, "arm64")) bundlePathAMD := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-%s-%s-0.0.1.tar.zst", bundleName, "amd64")) - tarballPath := filepath.Join("build", fmt.Sprintf("uds-bundle-%s-%s-0.0.1.tar.zst", bundleName, e2e.Arch)) + tarballPath := filepath.Join("build", bundleTarballName) // create and push bundles with different archs to the same OCI repo createLocal(t, bundleDir, "arm64") @@ -432,7 +437,7 @@ func TestBundleIndexInRemoteOnPublish(t *testing.T) { validateMultiArchIndex(t, index) inspectRemoteInsecure(t, fmt.Sprintf("oci://localhost:888/%s:0.0.1", bundleName)) - pull(t, fmt.Sprintf("localhost:888/%s:0.0.1", bundleName), tarballPath) // test no oci prefix + pull(t, fmt.Sprintf("localhost:888/%s:0.0.1", bundleName), bundleTarballName) // test no oci prefix deployAndRemoveLocalAndRemoteInsecure(t, fmt.Sprintf("oci://localhost:888/%s:0.0.1", bundleName), tarballPath) // now test by running 'create -o' over the bundle that was published @@ -441,17 +446,19 @@ func TestBundleIndexInRemoteOnPublish(t *testing.T) { require.NoError(t, err) validateMultiArchIndex(t, index) inspectRemoteInsecure(t, fmt.Sprintf("oci://localhost:888/%s:0.0.1", bundleName)) - pull(t, fmt.Sprintf("localhost:888/%s:0.0.1", bundleName), tarballPath) // test no oci prefix + pull(t, fmt.Sprintf("localhost:888/%s:0.0.1", bundleName), bundleTarballName) // test no oci prefix deployAndRemoveLocalAndRemoteInsecure(t, fmt.Sprintf("oci://localhost:888/%s:0.0.1", bundleName), tarballPath) } func TestBundleIndexInRemoteOnCreate(t *testing.T) { + deployZarfInit(t) e2e.SetupDockerRegistry(t, 888) defer e2e.TeardownRegistry(t, 888) bundleDir := "src/test/bundles/06-ghcr" bundleName := "ghcr-test" - tarballPath := filepath.Join("build", fmt.Sprintf("uds-bundle-%s-%s-0.0.1.tar.zst", bundleName, e2e.Arch)) + bundleTarballName := fmt.Sprintf("uds-bundle-%s-%s-0.0.1.tar.zst", bundleName, e2e.Arch) + tarballPath := filepath.Join("build", bundleTarballName) // create and push bundles with different archs to the same OCI repo createRemoteInsecure(t, bundleDir, "oci://localhost:888", "arm64") @@ -463,7 +470,7 @@ func TestBundleIndexInRemoteOnCreate(t *testing.T) { validateMultiArchIndex(t, index) inspectRemoteInsecure(t, fmt.Sprintf("oci://localhost:888/%s:0.0.1", bundleName)) - pull(t, fmt.Sprintf("localhost:888/%s:0.0.1", bundleName), tarballPath) // test no oci prefix + pull(t, fmt.Sprintf("localhost:888/%s:0.0.1", bundleName), bundleTarballName) // test no oci prefix deployAndRemoveLocalAndRemoteInsecure(t, fmt.Sprintf("oci://localhost:888/%s:0.0.1", bundleName), tarballPath) // now test by publishing over the bundle that was created with 'create -o' @@ -473,7 +480,7 @@ func TestBundleIndexInRemoteOnCreate(t *testing.T) { require.NoError(t, err) validateMultiArchIndex(t, index) inspectRemoteInsecure(t, fmt.Sprintf("oci://localhost:888/%s:0.0.1", bundleName)) - pull(t, fmt.Sprintf("localhost:888/%s:0.0.1", bundleName), tarballPath) // test no oci prefix + pull(t, fmt.Sprintf("localhost:888/%s:0.0.1", bundleName), bundleTarballName) // test no oci prefix deployAndRemoveLocalAndRemoteInsecure(t, fmt.Sprintf("oci://localhost:888/%s:0.0.1", bundleName), tarballPath) } @@ -498,6 +505,7 @@ func validateMultiArchIndex(t *testing.T, index ocispec.Index) { } func TestBundleWithComposedPkgComponent(t *testing.T) { + deployZarfInit(t) e2e.SetupDockerRegistry(t, 888) defer e2e.TeardownRegistry(t, 888) zarfPkgPath := "src/test/packages/prometheus" diff --git a/src/test/e2e/commands_test.go b/src/test/e2e/commands_test.go index b840f9cf..5a6aa280 100644 --- a/src/test/e2e/commands_test.go +++ b/src/test/e2e/commands_test.go @@ -205,7 +205,10 @@ func shasMatch(t *testing.T, path string, expected string) { require.Equal(t, expected, actual) } -func pull(t *testing.T, ref string, tarballPath string) { +func pull(t *testing.T, ref string, tarballName string) { + if !strings.HasSuffix(tarballName, "tar.zst") { + t.Fatalf("second arg to pull() must be the name a bundle tarball, got %s", tarballName) + } // todo: output somewhere other than build? cmd := strings.Split(fmt.Sprintf("pull %s -o build --insecure --oci-concurrency=10", ref), " ") _, _, err := e2e.UDS(cmd...) @@ -214,7 +217,7 @@ func pull(t *testing.T, ref string, tarballPath string) { decompressed := "build/decompressed-bundle" defer e2e.CleanFiles(decompressed) - cmd = []string{"zarf", "tools", "archiver", "decompress", tarballPath, decompressed} + cmd = []string{"zarf", "tools", "archiver", "decompress", filepath.Join("build", tarballName), decompressed} _, _, err = e2e.UDS(cmd...) require.NoError(t, err) diff --git a/src/test/e2e/dev_test.go b/src/test/e2e/dev_test.go index 4ff3254a..b0dbf759 100644 --- a/src/test/e2e/dev_test.go +++ b/src/test/e2e/dev_test.go @@ -71,4 +71,8 @@ func TestDevDeploy(t *testing.T) { require.NotContains(t, stderr, "This fun-fact was imported: Unicorns are the national animal of Scotland") remove(t, bundleTarballPath) }) + + // delete packages because other tests depend on them being created with SBOMs (ie. force other tests to re-create) + e2e.DeleteZarfPkg(t, "src/test/packages/podinfo") + e2e.DeleteZarfPkg(t, "src/test/packages/nginx") } diff --git a/src/test/e2e/ghcr_test.go b/src/test/e2e/ghcr_test.go index f4b3f152..a66b6e5b 100644 --- a/src/test/e2e/ghcr_test.go +++ b/src/test/e2e/ghcr_test.go @@ -27,7 +27,7 @@ func TestBundleCreateAndPublishGHCR(t *testing.T) { registryURL := "oci://ghcr.io/defenseunicorns/packages/uds-cli/test/publish" bundleGHCRPath := "defenseunicorns/packages/uds-cli/test/publish" - tarballPath := filepath.Join("build", fmt.Sprintf("uds-bundle-%s-%s-0.0.1.tar.zst", bundleName, e2e.Arch)) + bundleTarballName := fmt.Sprintf("uds-bundle-%s-%s-0.0.1.tar.zst", bundleName, e2e.Arch) bundleRef := registry.Reference{ Registry: registryURL, Repository: "ghcr-test", @@ -41,7 +41,7 @@ func TestBundleCreateAndPublishGHCR(t *testing.T) { registryURL = "ghcr.io/defenseunicorns/packages/uds-cli/test/publish" publish(t, bundlePathAMD, registryURL) inspectRemote(t, bundlePathARM) - pull(t, bundleRef.String(), tarballPath) + pull(t, bundleRef.String(), bundleTarballName) deploy(t, bundleRef.String()) remove(t, bundleRef.String()) diff --git a/src/test/e2e/main_test.go b/src/test/e2e/main_test.go index b02688ca..d8d9a685 100644 --- a/src/test/e2e/main_test.go +++ b/src/test/e2e/main_test.go @@ -91,6 +91,7 @@ func doAllTheThings(m *testing.M) (int, error) { return returnCode, nil } +// deployZarfInit deploys Zarf init (from a bundle!) if it hasn't already been deployed. func deployZarfInit(t *testing.T) { if !zarfInitDeployed() { // get Zarf version from go.mod diff --git a/src/test/e2e/optional_bundle_test.go b/src/test/e2e/optional_bundle_test.go new file mode 100644 index 00000000..4f53ceb1 --- /dev/null +++ b/src/test/e2e/optional_bundle_test.go @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2023-Present The UDS Authors + +// Package test provides e2e tests for UDS. +package test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/require" +) + +func TestBundleOptionalComponents(t *testing.T) { + deployZarfInit(t) + e2e.SetupDockerRegistry(t, 888) + defer e2e.TeardownRegistry(t, 888) + + // create 2 Zarf pkgs to be bundled + zarfPkgPath := "src/test/packages/podinfo-and-nginx" + e2e.CreateZarfPkg(t, zarfPkgPath, false) + + zarfPkgPath = "src/test/packages/prometheus" + pkg := filepath.Join(zarfPkgPath, fmt.Sprintf("zarf-package-prometheus-%s-0.0.1.tar.zst", e2e.Arch)) + e2e.CreateZarfPkg(t, zarfPkgPath, false) + zarfPublish(t, pkg, "localhost:888") + + // create bundle and publish + bundleDir := "src/test/bundles/14-optional-components" + bundleName := "optional-components" + bundleTarballName := fmt.Sprintf("uds-bundle-%s-%s-0.0.1.tar.zst", bundleName, e2e.Arch) + bundlePath := filepath.Join(bundleDir, bundleTarballName) + createLocal(t, bundleDir, e2e.Arch) + publishInsecure(t, bundlePath, "localhost:888") + + t.Run("look through contents of local bundle to ensure only selected components are present", func(t *testing.T) { + // local pkgs will have a correct pkg manifest (ie. missing non-selected optional component tarballs) + // remote pkgs will not, they will contain non-selected optional component tarballs + // because they already have a pkg manifest and we don't want to rewrite it + introspectOptionalComponentsBundle(t) + }) + + t.Run("test local deploy", func(t *testing.T) { + deploy(t, bundlePath) + remove(t, bundlePath) + }) + + t.Run("test remote deploy + pulled deploy", func(t *testing.T) { + pulledBundlePath := filepath.Join("build", bundleTarballName) + pull(t, fmt.Sprintf("localhost:888/%s:0.0.1", bundleName), bundleTarballName) + deployAndRemoveLocalAndRemoteInsecure(t, fmt.Sprintf("oci://localhost:888/%s:0.0.1", bundleName), pulledBundlePath) + }) +} + +// introspectOptionalComponentsBundle is a helper function that decompresses a bundle tarball and introspects the contents +// (has hardcoded checks meant for only the bundle in 14-optional-components) +func introspectOptionalComponentsBundle(t *testing.T) { + // ensure a decompressed bundle doesn't already exist + decompressionLoc := "build/decompressed-bundle" + err := os.RemoveAll(decompressionLoc) + if err != nil { + return + } + defer e2e.CleanFiles(decompressionLoc) + + // decompress the bundle + bundlePath := fmt.Sprintf("src/test/bundles/14-optional-components/uds-bundle-optional-components-%s-0.0.1.tar.zst", e2e.Arch) + cmd := []string{"zarf", "tools", "archiver", "decompress", bundlePath, decompressionLoc} + _, _, err = e2e.UDS(cmd...) + require.NoError(t, err) + + // read in the bundle's index.json + index := ocispec.Index{} + bundleIndexBytes, err := os.ReadFile(filepath.Join(decompressionLoc, "index.json")) + require.NoError(t, err) + err = json.Unmarshal(bundleIndexBytes, &index) + require.NoError(t, err) + require.Equal(t, 1, len(index.Manifests)) + blobsDir := filepath.Join(decompressionLoc, "blobs", "sha256") + + // grab the bundle root manifest + rootManifesBytes, err := os.ReadFile(filepath.Join(blobsDir, index.Manifests[0].Digest.Encoded())) + require.NoError(t, err) + bundleRootManifest := ocispec.Manifest{} + err = json.Unmarshal(rootManifesBytes, &bundleRootManifest) + require.NoError(t, err) + + // grab the first pkg (note that it came from a remote source) + pkgManifestBytes, err := os.ReadFile(filepath.Join(blobsDir, bundleRootManifest.Layers[0].Digest.Encoded())) + require.NoError(t, err) + remotePkgManifest := ocispec.Manifest{} + err = json.Unmarshal(pkgManifestBytes, &remotePkgManifest) + require.NoError(t, err) + + // ensure kiwix not present in bundle bc we didn't specify its component in the optional components + ensureImgNotPresent(t, "ghcr.io/kiwix/kiwix-serve", remotePkgManifest, blobsDir) + + // for this remote pkg, ensure component tars exist in img manifest, but not in the bundle + componentName := "optional-kiwix" + verifyComponentNotIncluded := false + for _, desc := range remotePkgManifest.Layers { + if strings.Contains(desc.Annotations[ocispec.AnnotationTitle], fmt.Sprintf("components/%s.tar", componentName)) { + _, err = os.ReadFile(filepath.Join(blobsDir, desc.Digest.Encoded())) + require.ErrorContains(t, err, "no such file or directory") + verifyComponentNotIncluded = true + } + } + require.True(t, verifyComponentNotIncluded) + + // grab the second pkg (note that it came from a local source) + pkgManifestBytes, err = os.ReadFile(filepath.Join(blobsDir, bundleRootManifest.Layers[1].Digest.Encoded())) + require.NoError(t, err) + localPkgManifest := ocispec.Manifest{} + err = json.Unmarshal(pkgManifestBytes, &localPkgManifest) + require.NoError(t, err) + + // ensure nginx not present in bundle bc we didn't specify its component in the optional components + ensureImgNotPresent(t, "docker.io/library/nginx", localPkgManifest, blobsDir) + + // for this local pkg, ensure component tars DO NOT exist in img manifest + componentName = "nginx-remote" + verifyComponentNotIncluded = true + for _, desc := range localPkgManifest.Layers { + if strings.Contains(desc.Annotations[ocispec.AnnotationTitle], fmt.Sprintf("components/%s.tar", componentName)) { + // component shouldn't exist in pkg manifest for locally sourced pkgs + verifyComponentNotIncluded = false + } + } + require.True(t, verifyComponentNotIncluded) +} + +func ensureImgNotPresent(t *testing.T, imgName string, remotePkgManifest ocispec.Manifest, blobsDir string) { + // used to verify that the kiwix img is not included in the bundle (note that kiwix is intentionally excluded!) + verifyImgNotIncluded := false + + // grab image index from pkg root manifest + var imgIndex ocispec.Index + for _, layer := range remotePkgManifest.Layers { + if layer.Annotations[ocispec.AnnotationTitle] == "images/index.json" { + imgIndexBytes, err := os.ReadFile(filepath.Join(blobsDir, layer.Digest.Encoded())) + require.NoError(t, err) + err = json.Unmarshal(imgIndexBytes, &imgIndex) + require.NoError(t, err) + + // ensure specified img exists in the img index but isn't actually included in the bundle + for _, desc := range imgIndex.Manifests { + if strings.Contains(desc.Annotations[ocispec.AnnotationBaseImageName], imgName) { + _, err = os.ReadFile(filepath.Join(blobsDir, desc.Digest.Encoded())) + require.ErrorContains(t, err, "no such file or directory") + verifyImgNotIncluded = true + break + } + } + break + } + } + require.True(t, verifyImgNotIncluded) +} diff --git a/src/test/packages/podinfo-and-nginx/zarf.yaml b/src/test/packages/podinfo-and-nginx/zarf.yaml new file mode 100644 index 00000000..09c8c62e --- /dev/null +++ b/src/test/packages/podinfo-and-nginx/zarf.yaml @@ -0,0 +1,19 @@ +kind: ZarfPackageConfig +metadata: + name: podinfo-and-nginx + description: used to test bundles with optional components + version: 0.0.1 + +components: + - name: podinfo + import: + path: ../podinfo + + - name: nginx-remote + import: + path: ../nginx + + - name: test + description: contains only single Zarf action + import: + path: ../no-cluster/real-simple diff --git a/src/test/packages/prometheus/zarf.yaml b/src/test/packages/prometheus/zarf.yaml index ca6af08c..fe4abddb 100644 --- a/src/test/packages/prometheus/zarf.yaml +++ b/src/test/packages/prometheus/zarf.yaml @@ -7,11 +7,21 @@ metadata: components: - name: upload-image - required: true description: test composition import: path: images name: upload + - name: optional-kiwix + description: used to test optional component images (not actually bundled) + charts: + # random helm chart that isn't important; just used to test that it doesn't get bundled + - name: prometheus-node-exporter + url: https://prometheus-community.github.io/helm-charts + version: 4.32.0 + namespace: prometheus + images: + # again, not bundled, we just need a unique name to test that it doesn't get bundled + - ghcr.io/kiwix/kiwix-serve:3.7.0 - name: deploy required: true charts: