From b60035d2117ade856b67b6d13531b16eee94aa8a Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Tue, 2 May 2023 17:11:25 -0400 Subject: [PATCH] copy.Options: add an InstancePlatforms field Add an InstancePlatforms field to the copy.Options structure. When asked to copy only a specific subset of instances, combine the Instances list with the instances which match the platforms given in the InstancePlatforms list, returning an error if we're unable to find an instance which matches any of the InstancePlatforms. Signed-off-by: Nalin Dahyabhai --- copy/copy.go | 6 +- copy/multiple.go | 39 +++++++++- copy/multiple_test.go | 170 ++++++++++++++++++++++++++++++++++++++++++ internal/set/set.go | 4 + 4 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 copy/multiple_test.go diff --git a/copy/copy.go b/copy/copy.go index bf8f4015b..1223e726b 100644 --- a/copy/copy.go +++ b/copy/copy.go @@ -23,6 +23,7 @@ import ( "github.com/containers/image/v5/types" encconfig "github.com/containers/ocicrypt/config" digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" "golang.org/x/exp/slices" "golang.org/x/sync/semaphore" @@ -91,8 +92,9 @@ type Options struct { PreserveDigests bool // manifest MIME type of image set by user. "" is default and means use the autodetection to the manifest MIME type ForceManifestMIMEType string - ImageListSelection ImageListSelection // set to either CopySystemImage (the default), CopyAllImages, or CopySpecificImages to control which instances we copy when the source reference is a list; ignored if the source reference is not a list - Instances []digest.Digest // if ImageListSelection is CopySpecificImages, copy only these instances and the list itself + ImageListSelection ImageListSelection // set to either CopySystemImage (the default), CopyAllImages, or CopySpecificImages to control which instances we copy when the source reference is a list; ignored if the source reference is not a list + Instances []digest.Digest // if ImageListSelection is CopySpecificImages, copy only these instances, instances matching the InstancePlatforms list, and the list itself + InstancePlatforms []imgspecv1.Platform // if ImageListSelection is CopySpecificImages, copy only matching instances, instances listed in the Instances list, and the list itself // Give priority to pulling gzip images if multiple images are present when configured to OptionalBoolTrue, // prefers the best compression if this is configured as OptionalBoolFalse. Choose automatically (and the choice may change over time) // if this is set to OptionalBoolUndefined (which is the default behavior, and recommended for most callers). diff --git a/copy/multiple.go b/copy/multiple.go index 097a18855..4bab4bc5a 100644 --- a/copy/multiple.go +++ b/copy/multiple.go @@ -10,11 +10,13 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/internal/image" internalManifest "github.com/containers/image/v5/internal/manifest" + "github.com/containers/image/v5/internal/set" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/signature" + "github.com/containers/image/v5/types" + digest "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" ) // copyMultipleImages copies some or all of an image list's instances, using @@ -89,15 +91,19 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur // Copy each image, or just the ones we want to copy, in turn. instanceDigests := updatedList.Instances() imagesToCopy := len(instanceDigests) + var specificImages *set.Set[digest.Digest] if options.ImageListSelection == CopySpecificImages { - imagesToCopy = len(options.Instances) + if specificImages, err = determineSpecificImages(options, updatedList); err != nil { + return nil, err + } + imagesToCopy = specificImages.Size() } c.Printf("Copying %d of %d images in list\n", imagesToCopy, len(instanceDigests)) updates := make([]manifest.ListUpdate, len(instanceDigests)) instancesCopied := 0 for i, instanceDigest := range instanceDigests { if options.ImageListSelection == CopySpecificImages && - !slices.Contains(options.Instances, instanceDigest) { + !specificImages.Contains(instanceDigest) { update, err := updatedList.Instance(instanceDigest) if err != nil { return nil, err @@ -196,3 +202,30 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur return manifestList, nil } + +// determineSpecificImages returns a set of images to copy based on the +// Instances and InstancePlatforms fields of the passed-in options structure +func determineSpecificImages(options *Options, updatedList internalManifest.List) (*set.Set[digest.Digest], error) { + // Start with the instances that were listed by digest. + specificImages := set.New[digest.Digest]() + for _, instanceDigest := range options.Instances { + specificImages.Add(instanceDigest) + } + if len(options.InstancePlatforms) > 0 { + // Choose the best match for each platform we were asked to + // also copy, and add it to the set of instances to copy. + for _, platform := range options.InstancePlatforms { + platformContext := types.SystemContext{ + OSChoice: platform.OS, + ArchitectureChoice: platform.Architecture, + VariantChoice: platform.Variant, + } + instanceDigest, err := updatedList.ChooseInstanceByCompression(&platformContext, options.PreferGzipInstances) + if err != nil { + return nil, fmt.Errorf("While choosing the instance for platform spec %v: %w", platform, err) + } + specificImages.Add(instanceDigest) + } + } + return specificImages, nil +} diff --git a/copy/multiple_test.go b/copy/multiple_test.go new file mode 100644 index 000000000..691cff622 --- /dev/null +++ b/copy/multiple_test.go @@ -0,0 +1,170 @@ +package copy + +import ( + "io/ioutil" + "strings" + "testing" + + internalManifest "github.com/containers/image/v5/internal/manifest" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDetermineSpecificImages(t *testing.T) { + testCases := []struct { + id string + fixture string + instanceDigests []digest.Digest + instancePlatforms []imgspecv1.Platform + expected []digest.Digest + expectedErrIncludes string + }{ + { + id: "no inputs no outputs", + fixture: "../manifest/fixtures/v2list.manifest.json", + }, + { + id: "instances only out of order", + fixture: "../manifest/fixtures/v2list.manifest.json", + instanceDigests: []digest.Digest{ + "sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2", + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + }, + expected: []digest.Digest{ + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + "sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2", + }, + }, + { + id: "instances only in order", + fixture: "../manifest/fixtures/v2list.manifest.json", + instanceDigests: []digest.Digest{ + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + "sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2", + }, + expected: []digest.Digest{ + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + "sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2", + }, + }, + { + id: "platforms only in order", + fixture: "../manifest/fixtures/v2list.manifest.json", + instancePlatforms: []imgspecv1.Platform{ + { + OS: "linux", + Architecture: "ppc64le", + }, + { + OS: "linux", + Architecture: "s390x", + }, + }, + expected: []digest.Digest{ + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + "sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2", + }, + }, + { + id: "platforms only out of order", + fixture: "../manifest/fixtures/v2list.manifest.json", + instancePlatforms: []imgspecv1.Platform{ + { + OS: "linux", + Architecture: "s390x", + }, + { + OS: "linux", + Architecture: "ppc64le", + }, + }, + expected: []digest.Digest{ + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + "sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2", + }, + }, + { + id: "mixed without duplicates", + fixture: "../manifest/fixtures/v2list.manifest.json", + instancePlatforms: []imgspecv1.Platform{ + { + OS: "linux", + Architecture: "s390x", + }, + }, + instanceDigests: []digest.Digest{ + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + }, + expected: []digest.Digest{ + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + "sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2", + }, + }, + { + id: "mixed with duplicates", + fixture: "../manifest/fixtures/v2list.manifest.json", + instancePlatforms: []imgspecv1.Platform{ + { + OS: "linux", + Architecture: "ppc64le", + }, + { + OS: "linux", + Architecture: "s390x", + }, + }, + instanceDigests: []digest.Digest{ + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + "sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2", + }, + expected: []digest.Digest{ + "sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783", + "sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2", + }, + }, + { + id: "no such platform", + fixture: "../manifest/fixtures/v2list.manifest.json", + instancePlatforms: []imgspecv1.Platform{ + { + OS: "windows", + Architecture: "amd64", + }, + { + OS: "darwin", + Architecture: "arm64", + }, + }, + expectedErrIncludes: "no image found in manifest list for", + }, + } + for _, tc := range testCases { + t.Run(tc.id, func(t *testing.T) { + listBlob, err := ioutil.ReadFile(tc.fixture) + require.NoErrorf(t, err, "unexpected error reading fixture %q", tc.fixture) + list, err := internalManifest.ListFromBlob(listBlob, internalManifest.GuessMIMEType(listBlob)) + require.NoErrorf(t, err, "unexpected error parsing fixture %q", tc.fixture) + options := Options{ + Instances: tc.instanceDigests, + InstancePlatforms: tc.instancePlatforms, + } + specific, err := determineSpecificImages(&options, list) + if err != nil && tc.expectedErrIncludes != "" { + if strings.Contains(err.Error(), tc.expectedErrIncludes) { + // okay + return + } + } + require.NoErrorf(t, err, "unexpected error selecting instances") + var selected []digest.Digest + for _, instanceDigest := range list.Instances() { + if specific.Contains(instanceDigest) { + selected = append(selected, instanceDigest) + } + } + assert.Equalf(t, tc.expected, selected, "given instance list %#v and platforms list %#v", tc.instanceDigests, tc.instancePlatforms) + }) + } +} diff --git a/internal/set/set.go b/internal/set/set.go index 5c7bcabef..a41ddc3ad 100644 --- a/internal/set/set.go +++ b/internal/set/set.go @@ -44,3 +44,7 @@ func (s *Set[E]) Empty() bool { func (s *Set[E]) Values() []E { return maps.Keys(s.m) } + +func (s *Set[E]) Size() int { + return len(s.m) +}